Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion vulnerabilities/api_v2.py
Original file line number Diff line number Diff line change
Expand Up @@ -788,7 +788,7 @@ def get_latest_run(self, schedule):

def to_representation(self, schedule):
representation = super().to_representation(schedule)
representation["run_interval"] = f"{schedule.run_interval}hr"
representation["run_interval"] = f"{schedule.run_interval}min"
representation["execution_timeout"] = f"{schedule.execution_timeout}hr"
return representation

Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,46 @@
# Generated by Django 5.2.11 on 2026-06-16 12:29

import django.core.validators
from django.db import migrations
from django.db import models
from django.db.models import F


class Migration(migrations.Migration):

dependencies = [
("vulnerabilities", "0136_advisorysetmember_unique_advisory_per_set"),
]

def convert_hours_to_minutes(apps, schema_editor):
PipelineSchedule = apps.get_model("vulnerabilities", "PipelineSchedule")
PipelineSchedule.objects.update(run_interval=F("run_interval") * 60)

def revert_convert_hours_to_minutes(apps, schema_editor):
PipelineSchedule = apps.get_model("vulnerabilities", "PipelineSchedule")
for schedule in PipelineSchedule.objects.all():
schedule.run_interval = min(8760, max(1, schedule.run_interval // 60))
schedule.save(update_fields=["run_interval"])

operations = [
migrations.AlterField(
model_name="pipelineschedule",
name="run_interval",
field=models.PositiveSmallIntegerField(
default=1440,
help_text="Number of minutes to wait between run of this pipeline.",
validators=[
django.core.validators.MinValueValidator(
5, message="Interval must be at least 5 minutes."
),
django.core.validators.MaxValueValidator(
43200, message="Interval must be at most 43200 minutes (i.e 30 days)."
),
],
),
),
migrations.RunPython(
convert_hours_to_minutes,
revert_convert_hours_to_minutes,
),
]
12 changes: 7 additions & 5 deletions vulnerabilities/models.py
Original file line number Diff line number Diff line change
Expand Up @@ -2302,13 +2302,15 @@ class ExecutionPriority(models.IntegerChoices):
),
)

run_interval = models.PositiveSmallIntegerField(
run_interval = models.IntegerField(
validators=[
MinValueValidator(1, message="Interval must be at least 1 hour."),
MaxValueValidator(8760, message="Interval must be at most 8760 hours."),
MinValueValidator(5, message="Interval must be at least 5 minutes."),
MaxValueValidator(
43200, message="Interval must be at most 43200 minutes (i.e 30 days)."
),
],
default=24,
help_text=("Number of hours to wait between run of this pipeline."),
default=1440,
help_text=("Number of minutes to wait between run of this pipeline."),
)

run_priority = models.IntegerField(
Expand Down
12 changes: 6 additions & 6 deletions vulnerabilities/pipelines/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -145,8 +145,8 @@ class VulnerableCodePipeline(PipelineDefinition, BasePipelineRun):
# When set to true pipeline is run only once.
# To rerun onetime pipeline reset is_active field to True via migration.
run_once = False
# Interval between runs in hour.
run_interval = 24
# Interval between runs in minutes.
run_interval = 1440
run_priority = PipelineSchedule.ExecutionPriority.DEFAULT

def on_failure(self):
Expand Down Expand Up @@ -180,8 +180,8 @@ class VulnerableCodeBaseImporterPipeline(VulnerableCodePipeline):
# When set to true pipeline is run only once.
# To rerun onetime pipeline reset is_active field to True via migration.
run_once = False
# Interval between runs in hour.
run_interval = 24
# Interval between runs in minutes.
run_interval = 1440
run_priority = PipelineSchedule.ExecutionPriority.DEFAULT

@classmethod
Expand Down Expand Up @@ -290,8 +290,8 @@ class VulnerableCodeBaseImporterPipelineV2(VulnerableCodePipeline):
# When set to true pipeline is run only once.
# To rerun onetime pipeline reset is_active field to True via migration.
run_once = False
# Interval between runs in hour.
run_interval = 24
# Interval between runs in minutes.
run_interval = 1440
run_priority = PipelineSchedule.ExecutionPriority.DEFAULT

@classmethod
Expand Down
5 changes: 5 additions & 0 deletions vulnerabilities/pipelines/v2_improvers/collect_ssvc_trees.py
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from vulnerabilities.models import SSVC
from vulnerabilities.models import AdvisorySeverity
from vulnerabilities.models import AdvisoryV2
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.severity_systems import SCORING_SYSTEMS

Expand All @@ -31,6 +32,10 @@ class CollectSSVCPipeline(VulnerableCodePipeline):

pipeline_id = "collect_ssvc_trees"

# Run pipeline every 30 minutes.
run_interval = 30
run_priority = PipelineSchedule.ExecutionPriority.HIGH

@classmethod
def steps(cls):
return (cls.collect_ssvc_data,)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
from vulnerabilities.models import AdvisorySeverity
from vulnerabilities.models import AdvisoryV2
from vulnerabilities.models import PackageV2
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.pipes.risk_score import bulk_update
from vulnerabilities.risk import compute_vulnerability_risk_factors
Expand All @@ -30,6 +31,10 @@ class ComputePackageRiskPipeline(VulnerableCodePipeline):
pipeline_id = "compute_package_risk_v2"
license_expression = None

# Run pipeline every 30 minutes.
run_interval = 30
run_priority = PipelineSchedule.ExecutionPriority.HIGH

@classmethod
def steps(cls):
return (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@
from dateutil import parser as dateparser

from vulnerabilities.models import AdvisoryExploit
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.utils import build_alias_to_advisory_map

Expand All @@ -30,6 +31,10 @@ class ExploitDBImproverPipeline(VulnerableCodePipeline):
pipeline_id = "enhance_with_exploitdb_v2"
spdx_license_expression = "GPL-2.0"

# Run pipeline every 30 minutes.
run_interval = 30
run_priority = PipelineSchedule.ExecutionPriority.HIGH

@classmethod
def steps(cls):
return (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -13,9 +13,7 @@
from aboutcode.pipeline import LoopProgress
from fetchcode.vcs import fetch_via_vcs

from vulnerabilities.models import AdvisoryAlias
from vulnerabilities.models import AdvisoryPOC
from vulnerabilities.models import AdvisoryV2
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.utils import relate_aliases_with_advisories

Expand Down
5 changes: 5 additions & 0 deletions vulnerabilities/pipelines/v2_improvers/enhance_with_kev.py
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
from aboutcode.pipeline import LoopProgress

from vulnerabilities.models import AdvisoryExploit
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.utils import build_alias_to_advisory_map

Expand All @@ -27,6 +28,10 @@ class VulnerabilityKevPipeline(VulnerableCodePipeline):
pipeline_id = "enhance_with_kev_v2"
license_expression = None

# Run pipeline every 30 minutes.
run_interval = 30
run_priority = PipelineSchedule.ExecutionPriority.HIGH

@classmethod
def steps(cls):
return (
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@
from dateutil import parser as dateparser

from vulnerabilities.models import AdvisoryExploit
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.utils import build_alias_to_advisory_map

Expand All @@ -29,6 +30,10 @@ class MetasploitImproverPipeline(VulnerableCodePipeline):
pipeline_id = "enhance_with_metasploit_v2"
spdx_license_expression = "BSD-3-clause"

# Run pipeline every 30 minutes.
run_interval = 30
run_priority = PipelineSchedule.ExecutionPriority.HIGH

@classmethod
def steps(cls):
return (
Expand Down
3 changes: 3 additions & 0 deletions vulnerabilities/pipelines/v2_improvers/flag_ghost_packages.py
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,9 @@ class FlagGhostPackagePipeline(VulnerableCodePipeline):

pipeline_id = "flag_ghost_packages_v2"

# Run pipeline every 6 hours.
run_interval = 360

@classmethod
def steps(cls):
return (cls.flag_ghost_packages,)
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -24,7 +24,6 @@
from vulnerabilities.models import PackageV2
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.pipes.group_advisories import group_advisory_for_package
from vulnerabilities.pipes.group_advisories import group_single_package_with_provided_advisories
from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk
from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS
Expand All @@ -37,7 +36,8 @@ class MarkUnfurlVersionRangePipeline(VulnerableCodePipeline):

pipeline_id = "mark_unfurl_version_range_v2"

run_interval = 1
# Run pipeline every 10 minutes.
run_interval = 10
run_priority = PipelineSchedule.ExecutionPriority.HIGH

@classmethod
Expand Down
5 changes: 5 additions & 0 deletions vulnerabilities/pipelines/v2_improvers/relate_severities.py
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@
from django.db import transaction

from vulnerabilities.models import AdvisoryV2
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.pipelines.v2_importers.epss_importer_v2 import EPSSImporterPipeline
from vulnerabilities.pipelines.v2_importers.suse_score_importer import (
Expand All @@ -35,6 +36,10 @@ class RelateSeveritiesPipeline(VulnerableCodePipeline):

pipeline_id = "relate_severities_v2"

# Run pipeline every 30 minutes.
run_interval = 30
run_priority = PipelineSchedule.ExecutionPriority.HIGH

# Severity systems to process
SUPPORTED_SYSTEMS = {
EPSS.identifier,
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -21,17 +21,12 @@
from univers.version_range import RANGE_CLASS_BY_SCHEMES
from univers.version_range import VersionRange

from vulnerabilities.models import AdvisoryV2
from vulnerabilities.models import ImpactedPackage
from vulnerabilities.models import ImpactedPackageAffecting
from vulnerabilities.models import ImpactedPackageFixedBy
from vulnerabilities.models import PackageV2
from vulnerabilities.models import PipelineSchedule
from vulnerabilities.pipelines import VulnerableCodePipeline
from vulnerabilities.pipes.fetchcode_utils import get_versions
from vulnerabilities.pipes.group_advisories import group_advisory_for_package
from vulnerabilities.pipes.risk_score import compute_package_risk_score_bulk
from vulnerabilities.utils import TYPES_WITH_MULTIPLE_IMPORTERS
from vulnerabilities.utils import update_purl_version


Expand All @@ -44,7 +39,8 @@ class UnfurlVersionRangePipeline(VulnerableCodePipeline):

pipeline_id = "unfurl_version_range_v2"

run_interval = 1
# Run pipeline every 10 minutes.
run_interval = 10
run_priority = PipelineSchedule.ExecutionPriority.HIGH

# Days elapsed before version range is re-unfurled
Expand Down
11 changes: 6 additions & 5 deletions vulnerabilities/schedules.py
Original file line number Diff line number Diff line change
Expand Up @@ -29,7 +29,7 @@ def schedule_execution(pipeline_schedule, execute_now=False):
if not execute_now:
first_execution = pipeline_schedule.next_run_date

interval_in_seconds = pipeline_schedule.run_interval * 60 * 60
interval_in_seconds = pipeline_schedule.run_interval * 60

job = scheduler.schedule(
scheduled_time=first_execution,
Expand Down Expand Up @@ -97,7 +97,7 @@ def update_pipeline_schedule():
PipelineSchedule.objects.exclude(pipeline_id__in=pipelines.keys()).delete()
for id, pipeline_class in pipelines.items():
run_once = getattr(pipeline_class, "run_once", False)
run_interval = getattr(pipeline_class, "run_interval", 24)
run_interval = getattr(pipeline_class, "run_interval", 1440)
run_priority = getattr(
pipeline_class, "run_priority", PipelineSchedule.ExecutionPriority.DEFAULT
)
Expand All @@ -112,6 +112,7 @@ def update_pipeline_schedule():
)

if not created:
pipeline.run_priority = run_priority
pipeline.run_interval = run_interval
pipeline.save()
if pipeline.run_priority != run_priority or pipeline.run_interval != run_interval:
pipeline.run_priority = run_priority
pipeline.run_interval = run_interval
pipeline.save()
2 changes: 1 addition & 1 deletion vulnerabilities/tasks.py
Original file line number Diff line number Diff line change
Expand Up @@ -179,7 +179,7 @@ def compute_queue_load_factor():

for queue in RQ_QUEUES.keys():
total_compute_seconds_per_queue[queue] = sum(
(p.latest_successful_run.runtime / (p.run_interval / 24))
(p.latest_successful_run.runtime / (p.run_interval / (24 * 60)))
for p in models.PipelineSchedule.objects.filter(
is_active=True, run_priority=label_to_value[queue]
)
Expand Down
34 changes: 17 additions & 17 deletions vulnerabilities/templates/pipeline_dashboard.html
Original file line number Diff line number Diff line change
Expand Up @@ -45,7 +45,7 @@
<div class="column">
</div>

<div class="column is-four-fifths">
<div class="column is-11">
<div class="content is-normal">
<h1>Pipeline Dashboard</h1>
<hr />
Expand Down Expand Up @@ -126,13 +126,13 @@ <h1>Pipeline Dashboard</h1>
<th colspan="6">
<div class="box is-small">
<div class="columns is-mobile is-vcentered">
<div class="column is-one-quarter">Pipeline ID</div>
<div class="column is-one-eighth">Active</div>
<div class="column is-one-eighth">Priority</div>
<div class="column is-one-eighth">Interval</div>
<div class="column is-one-eighth">Status</div>
<div class="column is-one-fifth">Last Run End Time</div>
<div class="column is-one-fifth">Next Run Start</div>
<div class="column has-text-left" style="flex: 0 0 25%; font-weight: bold;">Pipeline ID</div>
<div class="column has-text-left" style="flex: 0 0 8%; font-weight: bold;">Active</div>
<div class="column has-text-left" style="flex: 0 0 8%; font-weight: bold;">Priority</div>
<div class="column has-text-left" style="flex: 0 0 13%; font-weight: bold;">Interval</div>
<div class="column has-text-left" style="flex: 0 0 14%; font-weight: bold;">Status</div>
<div class="column has-text-left" style="flex: 0 0 18%; font-weight: bold;">Last Run End Time</div>
<div class="column has-text-left" style="flex: 0 0 14%; font-weight: bold;">Next Run Start</div>
</div>
</div>
</th>
Expand All @@ -143,30 +143,30 @@ <h1>Pipeline Dashboard</h1>
<tr>
<td colspan="6">
<a href="{% url 'runs-list' pipeline_id=schedule.pipeline_id %}" class="has-text-info">
<div class="columns px-1 is-mobile is-vcentered">
<div class="column is-one-quarter">{{ schedule.pipeline_id }}</div>
<div class="column is-one-eighth has-text-grey">{{ schedule.is_active|yesno:"Yes,No" }}</div>
<div class="column is-one-eighth has-text-grey">{{ schedule.get_run_priority_display|capfirst}}</div>
<div class="column is-one-eighth has-text-grey">
<div class="columns mx-2 is-mobile is-vcentered">
<div class="colum has-text-left" style="flex: 0 0 25%;">{{ schedule.pipeline_id }}</div>
<div class="column has-text-grey" style="flex: 0 0 8%;">{{ schedule.is_active|yesno:"Yes,No" }}</div>
<div class="column has-text-grey" style="flex: 0 0 8%;">{{ schedule.get_run_priority_display|capfirst}}</div>
<div class="column has-text-grey" style="flex: 0 0 13%;">
{% if schedule.is_run_once %}
Once
{% else %}
{{ schedule.run_interval }} hour{{ schedule.run_interval|pluralize }}
{{ schedule.run_interval|humanize_interval }}
{% endif %}
</div>
<div class="column is-one-eighth">
<div class="column" style="flex: 0 0 14%;">
<span class="is-flex is-align-items-center">
{% include "includes/job_status.html" with status=schedule.status %}
</span>
</div>
<div class="column is-one-fifth has-text-grey">
<div class="column has-text-grey" style="flex: 0 0 18%;">
{% if schedule.latest_run_end_date %}
{{ schedule.latest_run_end_date|date:"Y-m-d h:i a T" }}
{% else %}
N/A
{% endif %}
</div>
<div class="column is-one-fifth has-text-grey">
<div class="column has-text-grey" style="flex: 0 0 14%;">
{% if schedule.next_run_date %}
{{ schedule.next_run_date|date:"Y-m-d" }}
{% else %}
Expand Down
Loading