Skip to content

Commit 1a4b7d2

Browse files
0ssigenocode-review-doctor[bot]drosetti
authored
Refactor pivot (#1902)
* Refactor pivot Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * More Signed-off-by: 0ssigeno <s.berni@certego.net> * Linters Signed-off-by: 0ssigeno <s.berni@certego.net> * Update api_app/pivots_manager/models.py Co-authored-by: code-review-doctor[bot] <72320148+code-review-doctor[bot]@users.noreply.github.com> * Fix Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Deepsource Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix Signed-off-by: 0ssigeno <s.berni@certego.net> * fix test frontend * Prettier Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * Blake Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix typo Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix Signed-off-by: 0ssigeno <s.berni@certego.net> * improved pivot frontend * frontend * Fix Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix url property Signed-off-by: 0ssigeno <s.berni@certego.net> * Fix url property Signed-off-by: 0ssigeno <s.berni@certego.net> * improved pivoting page * Removed import Signed-off-by: 0ssigeno <s.berni@certego.net> * prettier * Fixes Signed-off-by: 0ssigeno <s.berni@certego.net> * fixed frontend tests * prettier * Fix yaraify url Signed-off-by: 0ssigeno <s.berni@certego.net> * Adding removal of existent directory Signed-off-by: 0ssigeno <s.berni@certego.net> --------- Signed-off-by: 0ssigeno <s.berni@certego.net> Co-authored-by: code-review-doctor[bot] <72320148+code-review-doctor[bot]@users.noreply.github.com> Co-authored-by: Daniele Rosetti <d.rosetti@certego.net>
1 parent 5ccca8c commit 1a4b7d2

File tree

78 files changed

+3399
-2477
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

78 files changed

+3399
-2477
lines changed

‎api_app/admin.py

+1-3
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
from django.db.models import JSONField
66
from prettyjson.widgets import PrettyJSONWidget
77

8-
from .forms import ParameterInlineForm, PythonConfigAdminForm
8+
from .forms import ParameterInlineForm
99
from .models import AbstractConfig, Job, Parameter, PluginConfig, PythonModule, Tag
1010
from .tabulars import (
1111
ParameterInline,
@@ -148,8 +148,6 @@ def disabled_in_orgs(self, instance: AbstractConfig):
148148

149149

150150
class PythonConfigAdminView(AbstractConfigAdminView):
151-
form = PythonConfigAdminForm
152-
153151
list_display = (
154152
"name",
155153
"python_module",

‎api_app/analyzers_manager/admin.py

-2
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,6 @@
33
from django.contrib import admin
44

55
from api_app.admin import AbstractReportAdminView, PythonConfigAdminView
6-
from api_app.analyzers_manager.forms import AnalyzerConfigAdminForm
76
from api_app.analyzers_manager.models import AnalyzerConfig, AnalyzerReport
87

98

@@ -20,5 +19,4 @@ class AnalyzerConfigAdminView(PythonConfigAdminView):
2019
"maximum_tlp",
2120
"update_schedule",
2221
)
23-
form = AnalyzerConfigAdminForm
2422
exclude = ["update_task"]

‎api_app/analyzers_manager/file_analyzers/yara_scan.py

+4-3
Original file line numberDiff line numberDiff line change
@@ -158,9 +158,10 @@ def compiled_file_name(self):
158158
@cached_property
159159
def first_level_directories(self) -> List[PosixPath]:
160160
paths = []
161-
for directory in self.directory.iterdir():
162-
if directory.is_dir() and directory.stem not in [".git", ".github"]:
163-
paths.append(directory)
161+
if self.directory.exists():
162+
for directory in self.directory.iterdir():
163+
if directory.is_dir() and directory.stem not in [".git", ".github"]:
164+
paths.append(directory)
164165
return paths
165166

166167
@cached_property

‎api_app/analyzers_manager/forms.py

-10
This file was deleted.

‎api_app/analyzers_manager/migrations/0042_alter_analyzerconfig_python_module.py

+32
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
# Generated by Django 4.1.10 on 2023-09-14 13:25
2+
3+
import django.db.models.deletion
4+
from django.db import migrations, models
5+
6+
7+
class Migration(migrations.Migration):
8+
dependencies = [
9+
(
10+
"api_app",
11+
"0046_remove_pluginconfig_plugin_config_no_config_all_null_and_more",
12+
),
13+
("analyzers_manager", "0041_alter_analyzerconfig_python_module"),
14+
]
15+
16+
operations = [
17+
migrations.AlterField(
18+
model_name="analyzerconfig",
19+
name="python_module",
20+
field=models.ForeignKey(
21+
limit_choices_to={
22+
"base_path__in": [
23+
"api_app.analyzers_manager.file_analyzers",
24+
"api_app.analyzers_manager.observable_analyzers",
25+
]
26+
},
27+
on_delete=django.db.models.deletion.PROTECT,
28+
related_name="%(class)ss",
29+
to="api_app.pythonmodule",
30+
),
31+
),
32+
]

‎api_app/analyzers_manager/migrations/0043_modify_yaraify_url.py

+38
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
# Generated by Django 4.1.10 on 2023-09-14 13:25
2+
import shutil
3+
from pathlib import PosixPath
4+
5+
from django.db import migrations
6+
7+
8+
def migrate(apps, schema_editor):
9+
PluginConfig = apps.get_model("api_app", "PluginConfig")
10+
pc = PluginConfig.objects.get(
11+
parameter__name="repositories",
12+
owner=None,
13+
for_organization=False,
14+
parameter__python_module__module="yara_scan.YaraScan",
15+
)
16+
pc.value.remove("https://yaraify-api.abuse.ch/download/yaraify-rules.zip")
17+
path = PosixPath(
18+
"/opt/deploy/files_required/yara/yaraify-api.abuse.ch_yaraify-rules"
19+
)
20+
if path.exists():
21+
shutil.rmtree(str(path), ignore_errors=True)
22+
23+
pc.value.append("https://yaraify-api.abuse.ch/yarahub/yaraify-rules.zip")
24+
pc.save()
25+
26+
27+
def reverse_migrate(apps, schema_editor):
28+
pass
29+
30+
31+
class Migration(migrations.Migration):
32+
dependencies = [
33+
("analyzers_manager", "0042_alter_analyzerconfig_python_module"),
34+
]
35+
36+
operations = [
37+
migrations.RunPython(migrate, reverse_migrate),
38+
]

‎api_app/analyzers_manager/models.py

+13-2
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@
1414
TypeChoices,
1515
)
1616
from api_app.analyzers_manager.exceptions import AnalyzerConfigurationException
17-
from api_app.choices import TLP
17+
from api_app.choices import TLP, PythonModuleBasePaths
1818
from api_app.fields import ChoiceArrayField
19-
from api_app.models import AbstractReport, PythonConfig
19+
from api_app.models import AbstractReport, PythonConfig, PythonModule
2020

2121
logger = getLogger(__name__)
2222

@@ -131,6 +131,17 @@ class AnalyzerConfig(PythonConfig):
131131
maximum_tlp = models.CharField(
132132
null=False, default=TLP.RED, choices=TLP.choices, max_length=50
133133
)
134+
python_module = models.ForeignKey(
135+
PythonModule,
136+
on_delete=models.PROTECT,
137+
related_name="%(class)ss",
138+
limit_choices_to={
139+
"base_path__in": [
140+
PythonModuleBasePaths.FileAnalyzer.value,
141+
PythonModuleBasePaths.ObservableAnalyzer.value,
142+
]
143+
},
144+
)
134145
# obs
135146
observable_supported = ChoiceArrayField(
136147
models.CharField(null=False, choices=ObservableTypes.choices, max_length=30),

‎api_app/analyzers_manager/serializers.py

-4
Original file line numberDiff line numberDiff line change
@@ -12,10 +12,6 @@
1212

1313

1414
class AnalyzerReportSerializer(AbstractReportSerializer):
15-
"""
16-
AnalyzerReport model's serializer.
17-
"""
18-
1915
class Meta:
2016
model = AnalyzerReport
2117
fields = AbstractReportSerializer.Meta.fields

‎api_app/choices.py

+19-13
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
# This file is a part of IntelOwl https://github.com/intelowlproject/IntelOwl
22
# See the file 'LICENSE' for copying permission.
3+
import enum
34
import typing
45
from pathlib import PosixPath
56

@@ -19,6 +20,7 @@ class PythonModuleBasePaths(models.TextChoices):
1920
Connector = PosixPath("api_app.connectors_manager.connectors"), "Connector"
2021
Ingestor = PosixPath("api_app.ingestors_manager.ingestors"), "Ingestor"
2122
Visualizer = PosixPath("api_app.visualizers_manager.visualizers"), "Visualizer"
23+
Pivot = PosixPath("api_app.pivots_manager.pivots"), "Pivot"
2224

2325

2426
class TLP(models.TextChoices):
@@ -55,33 +57,37 @@ class Status(models.TextChoices):
5557
RUNNING = "running", "running"
5658

5759
ANALYZERS_RUNNING = "analyzers_running", "analyzers_running"
58-
CONNECTORS_RUNNING = "connectors_running", "connectors_running"
59-
VISUALIZERS_RUNNING = "visualizers_running", "visualizers_running"
60-
6160
ANALYZERS_COMPLETED = "analyzers_completed", "analyzers_completed"
61+
62+
CONNECTORS_RUNNING = "connectors_running", "connectors_running"
6263
CONNECTORS_COMPLETED = "connectors_completed", "connectors_completed"
64+
65+
PIVOTS_RUNNING = "pivots_running", "pivots_running"
66+
PIVOTS_COMPLETED = "pivots_completed", "pivots_completed"
67+
68+
VISUALIZERS_RUNNING = "visualizers_running", "visualizers_running"
6369
VISUALIZERS_COMPLETED = "visualizers_completed", "visualizers_completed"
6470

6571
REPORTED_WITHOUT_FAILS = "reported_without_fails", "reported_without_fails"
6672
REPORTED_WITH_FAILS = "reported_with_fails", "reported_with_fails"
6773
KILLED = "killed", "killed"
6874
FAILED = "failed", "failed"
6975

76+
@classmethod
77+
def get_enums_with_suffix(
78+
cls, suffix: str
79+
) -> typing.Generator[enum.Enum, None, None]:
80+
for key in cls:
81+
if key.name.endswith(suffix):
82+
yield key
83+
7084
@classmethod
7185
def running_statuses(cls) -> typing.List["Status"]:
72-
return [
73-
cls.ANALYZERS_RUNNING,
74-
cls.CONNECTORS_RUNNING,
75-
cls.VISUALIZERS_RUNNING,
76-
]
86+
return list(cls.get_enums_with_suffix("_RUNNING"))
7787

7888
@classmethod
7989
def partial_statuses(cls) -> typing.List["Status"]:
80-
return [
81-
cls.ANALYZERS_COMPLETED,
82-
cls.CONNECTORS_COMPLETED,
83-
cls.VISUALIZERS_COMPLETED,
84-
]
90+
return list(cls.get_enums_with_suffix("_COMPLETED"))
8591

8692
@classmethod
8793
def final_statuses(cls) -> typing.List["Status"]:

‎api_app/classes.py

+1-12
Original file line numberDiff line numberDiff line change
@@ -104,23 +104,13 @@ def after_run(self):
104104
self.report.end_time = timezone.now()
105105
self.report.save()
106106

107-
def after_run_success(self, content: typing.Union[typing.Dict, typing.Iterable]):
107+
def after_run_success(self, content: typing.Any):
108108
if isinstance(content, typing.Generator):
109109
content = list(content)
110110
self.report.report = content
111111
self.report.status = self.report.Status.SUCCESS.value
112112
self.report.save(update_fields=["status", "report"])
113113

114-
def execute_pivots(self) -> None:
115-
from api_app.pivots_manager.models import PivotConfig
116-
117-
if self._job.playbook_to_execute:
118-
for pivot in self._config.pivots.annotate_runnable(self._job.user).filter(
119-
used_by_playbooks=self._job.playbook_to_execute, runnable=True
120-
):
121-
pivot: PivotConfig
122-
pivot.pivot_job(self._job)
123-
124114
def log_error(self, e):
125115
if isinstance(e, (*self.get_exceptions_to_catch(), SoftTimeLimitExceeded)):
126116
error_message = self.get_error_message(e)
@@ -204,7 +194,6 @@ def start(self, *args, **kwargs):
204194
self.after_run_failed(e)
205195
else:
206196
self.after_run_success(_result)
207-
self.execute_pivots()
208197
finally:
209198
# add end time of process
210199
self.after_run()

‎api_app/connectors_manager/admin.py

-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
from django.contrib import admin
55

66
from api_app.admin import AbstractReportAdminView, PythonConfigAdminView
7-
from api_app.connectors_manager.forms import ConnectorConfigAdminForm
87
from api_app.connectors_manager.models import ConnectorConfig, ConnectorReport
98

109

@@ -19,4 +18,3 @@ class ConnectorConfigAdminView(PythonConfigAdminView):
1918
"maximum_tlp",
2019
"run_on_failure",
2120
)
22-
form = ConnectorConfigAdminForm

‎api_app/connectors_manager/forms.py

-7
Original file line numberDiff line numberDiff line change
@@ -1,7 +0,0 @@
1-
from api_app.choices import PythonModuleBasePaths
2-
from api_app.forms import PythonConfigAdminForm
3-
4-
5-
class ConnectorConfigAdminForm(PythonConfigAdminForm):
6-
class Meta:
7-
base_paths_allowed = [PythonModuleBasePaths.Connector.value]

‎api_app/connectors_manager/migrations/0022_alter_connectorconfig_python_module.py

+27
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
# Generated by Django 4.1.10 on 2023-09-14 13:25
2+
3+
import django.db.models.deletion
4+
from django.db import migrations, models
5+
6+
7+
class Migration(migrations.Migration):
8+
dependencies = [
9+
(
10+
"api_app",
11+
"0046_remove_pluginconfig_plugin_config_no_config_all_null_and_more",
12+
),
13+
("connectors_manager", "0021_alter_connectorconfig_python_module"),
14+
]
15+
16+
operations = [
17+
migrations.AlterField(
18+
model_name="connectorconfig",
19+
name="python_module",
20+
field=models.ForeignKey(
21+
limit_choices_to={"base_path": "api_app.connectors_manager.connectors"},
22+
on_delete=django.db.models.deletion.PROTECT,
23+
related_name="%(class)ss",
24+
to="api_app.pythonmodule",
25+
),
26+
),
27+
]

‎api_app/connectors_manager/models.py

+8-2
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,9 @@
33

44
from django.db import models
55

6-
from api_app.choices import TLP
6+
from api_app.choices import TLP, PythonModuleBasePaths
77
from api_app.connectors_manager.exceptions import ConnectorConfigurationException
8-
from api_app.models import AbstractReport, PythonConfig
8+
from api_app.models import AbstractReport, PythonConfig, PythonModule
99

1010

1111
class ConnectorReport(AbstractReport):
@@ -22,6 +22,12 @@ class ConnectorConfig(PythonConfig):
2222
null=False, default=TLP.CLEAR, choices=TLP.choices, max_length=50
2323
)
2424
run_on_failure = models.BooleanField(null=False, default=True)
25+
python_module = models.ForeignKey(
26+
PythonModule,
27+
on_delete=models.PROTECT,
28+
related_name="%(class)ss",
29+
limit_choices_to={"base_path": PythonModuleBasePaths.Connector.value},
30+
)
2531

2632
@classmethod
2733
@property

‎api_app/defaults.py

+1
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ def default_runtime():
1111
return {
1212
"analyzers": {},
1313
"connectors": {},
14+
"pivots": {},
1415
"visualizers": {},
1516
}
1617

‎api_app/forms.py

+1-16
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
from django import forms
2-
from django.forms.models import ALL_FIELDS
32

4-
from api_app.models import Parameter, PythonConfig
3+
from api_app.models import Parameter
54

65

76
class MultilineJSONField(forms.JSONField):
@@ -40,17 +39,3 @@ class Meta:
4039
"required",
4140
"python_module",
4241
]
43-
44-
45-
class PythonConfigAdminForm(forms.ModelForm):
46-
class Meta:
47-
model = PythonConfig
48-
fields = ALL_FIELDS
49-
base_paths_allowed = []
50-
51-
def __init__(self, *args, **kwargs):
52-
super().__init__(*args, **kwargs)
53-
# only modules of this configurations
54-
self.fields["python_module"].queryset = self.fields[
55-
"python_module"
56-
].queryset.filter(base_path__in=self.Meta.base_paths_allowed)

‎api_app/ingestors_manager/admin.py

-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
from django.contrib import admin
55

66
from api_app.admin import AbstractReportAdminView, PythonConfigAdminView
7-
from api_app.ingestors_manager.forms import IngestorConfigAdminForm
87
from api_app.ingestors_manager.models import IngestorConfig, IngestorReport
98

109

@@ -23,4 +22,3 @@ class IngestorConfigAdminView(PythonConfigAdminView):
2322
"schedule",
2423
)
2524
exclude = ["user", "periodic_task"]
26-
form = IngestorConfigAdminForm

‎api_app/ingestors_manager/classes.py

+1-1
Original file line numberDiff line numberDiff line change
@@ -65,7 +65,7 @@ def after_run_success(self, content):
6565
deque(
6666
self._config._create_jobs(
6767
# every job created from an ingestor
68-
self.report,
68+
content,
6969
TLP.CLEAR.value,
7070
self._user,
7171
),

0 commit comments

Comments
 (0)