Skip to content

Level 2: replace CodeQL with PyCG, add coupling-aware adaptive sharding #50

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
rahlk merged 14 commits into from
Jun 27, 2026
Merged

Level 2: replace CodeQL with PyCG, add coupling-aware adaptive sharding #50

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
60be312
Remove CodeQL backend; introduce analysis_level option
sinha108 Jun 18, 2026
aab32e8
Add PyCG as Level 2 call graph backend
sinha108 Jun 18, 2026
9eea71f
Add PyCG sharding for level 2 analysis on large projects
sinha108 Jun 23, 2026
7b78484
Restore Jedi call graph at level 1; filter lib→lib edges
sinha108 Jun 24, 2026
c85c457
Merge main into feat/level2-pycg-remove-codeql
sinha108 Jun 24, 2026
0fd09b4
Remove CodeQL re-introduced by merge with main
sinha108 Jun 24, 2026
234b7ee
Fix tests broken by merge and level-1 Jedi restoration
sinha108 Jun 24, 2026
10afac1
Add Ray-parallel PyCG shard processing
sinha108 Jun 24, 2026
0799828
Add timing/progress, fix double-bar render, fix venv warn-and-continue
sinha108 Jun 25, 2026
6916884
feat(pycg): coupling-aware shard planning from the Jedi module graph
rahlk Jun 27, 2026
5c02ba3
feat(pycg): Ray-parallel execution for Jedi-planned shards
rahlk Jun 27, 2026
3985d69
fix(pycg): key the shard module graph by file path, not module_name
rahlk Jun 27, 2026
81fd409
feat(pycg): bound the fixpoint with --pycg-max-iter; stop following i…
rahlk Jun 27, 2026
f72ef00
feat(pycg): iterative decomposition of runaway shards
rahlk Jun 27, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
96 changes: 90 additions & 6 deletions codeanalyzer/__main__.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@
from codeanalyzer.utils import _set_log_level, logger
from codeanalyzer.config import OutputFormat
from codeanalyzer.schema import model_dump_json
from codeanalyzer.options import AnalysisOptions, EmitTarget
from codeanalyzer.options import AnalysisOptions, EmitTarget, ShardStrategy


def main(
Expand Down Expand Up @@ -83,9 +83,16 @@ def main(
help="Neo4j database name (default: server default). [env: NEO4J_DATABASE]",
),
] = None,
using_codeql: Annotated[
bool, typer.Option("--codeql/--no-codeql", help="Enable CodeQL-based analysis.")
] = False,
analysis_level: Annotated[
int,
typer.Option(
"-a",
"--analysis-level",
help="Analysis depth: 1=symbol table+Jedi call graph, 2=+PyCG call graph.",
min=1,
max=2,
),
] = 1,
using_ray: Annotated[
bool,
typer.Option("--ray/--no-ray", help="Enable Ray for distributed analysis."),
Expand Down Expand Up @@ -137,6 +144,78 @@ def main(
verbosity: Annotated[
int, typer.Option("-v", count=True, help="Increase verbosity: -v, -vv, -vvv")
] = 0,
pycg_shard: Annotated[
bool,
typer.Option(
"--pycg-shard/--no-pycg-shard",
help=(
"Shard PyCG call-graph analysis by Python package (level 2 only). "
"When the project exceeds the 500-file ceiling, PyCG is run "
"independently per top-level package with cross-package imports "
"treated as ghost nodes. Without this flag, projects over the "
"ceiling fall back to Jedi-only edges."
),
),
] = False,
pycg_shard_ceiling: Annotated[
int,
typer.Option(
"--pycg-shard-ceiling",
help=(
"Maximum files per shard when --pycg-shard is active (default 100). "
"Shards exceeding this limit are skipped; their call edges are "
"omitted from the call graph (Jedi edges for those packages are "
"still included). Lower values are safer for packages with deep "
"class hierarchies or heavy import graphs."
),
min=1,
),
] = 100,
pycg_shard_timeout: Annotated[
int,
typer.Option(
"--pycg-shard-timeout",
help=(
"Per-shard wall-clock timeout in seconds when --pycg-shard is "
"active (default 120). A shard that exceeds this limit is skipped "
"gracefully. PyCG's fixpoint is bimodal: it either converges "
"quickly or diverges indefinitely, so the timeout acts as a final "
"safety net after the file-count ceiling. Set to 0 to disable. "
"POSIX only (macOS / Linux); ignored on Windows."
),
min=0,
),
] = 120,
pycg_shard_strategy: Annotated[
ShardStrategy,
typer.Option(
"--pycg-shard-strategy",
help=(
"How --pycg-shard groups files (level 2 only). 'jedi' (default) "
"partitions the Jedi module-dependency graph (SCC + Louvain) so "
"tightly-coupled modules co-compute and few call edges are "
"severed between shards; import cycles are never split. "
"'package' uses the legacy one-shard-per-package-directory "
"grouping."
),
),
] = ShardStrategy.JEDI,
pycg_max_iter: Annotated[
int,
typer.Option(
"--pycg-max-iter",
help=(
"Cap on PyCG's fixpoint passes per shard/project (level 2; "
"default 50). PyCG iterates until its points-to state stops "
"changing, but its access-path domain has no convergence bound, "
"so heavy metaclass/mixin code (e.g. an ORM) can loop with each "
"pass costing seconds. The cap returns a sound-but-incomplete "
"call graph instead of looping until the timeout kills it. "
"Set to -1 for PyCG's unbounded run-to-convergence behaviour."
),
min=-1,
),
] = 50,
):
options = AnalysisOptions(
input=input,
Expand All @@ -148,7 +227,7 @@ def main(
neo4j_user=neo4j_user,
neo4j_password=neo4j_password,
neo4j_database=neo4j_database,
using_codeql=using_codeql,
analysis_level=analysis_level,
using_ray=using_ray,
rebuild_analysis=rebuild_analysis,
skip_tests=skip_tests,
Expand All @@ -157,6 +236,11 @@ def main(
cache_dir=cache_dir,
clear_cache=clear_cache,
verbosity=verbosity,
pycg_shard=pycg_shard,
pycg_shard_ceiling=pycg_shard_ceiling,
pycg_shard_timeout=pycg_shard_timeout,
pycg_shard_strategy=pycg_shard_strategy,
pycg_max_iter=pycg_max_iter,
)

_set_log_level(options.verbosity)
Expand Down Expand Up @@ -230,7 +314,7 @@ def _write_output(artifacts, output_dir: Path, format: OutputFormat):
app = typer.Typer(
callback=main,
name="canpy",
help="Static Analysis on Python source code using Jedi, CodeQL and Tree sitter.",
help="Static Analysis on Python source code using Jedi, PyCG and Tree sitter.",
invoke_without_command=True,
no_args_is_help=True,
add_completion=False,
Expand Down
Loading