Skip to content

Commit

Permalink
Browse files Browse the repository at this point in the history
  • Loading branch information
rkorytkowski committed Jan 22, 2025
1 parent 3595189 commit 93a51d7
Showing 1 changed file with 13 additions and 9 deletions.
22 changes: 13 additions & 9 deletions core/importers/importer.py
Original file line number Diff line number Diff line change
Expand Up @@ -375,15 +375,7 @@ def traverse_dependencies(self, package_file, path, resource_types, dependencies

def prepare_tasks(self, resource_types, packages, resources):
tasks = []
# Count all items to determine batch size
all_count = 0
for resource, item in resources.items():
for filepath, count in item.items():
all_count += count
if all_count > 50000:
task_batch_size = (all_count / 1000)
else:
task_batch_size = self.MIN_BATCH_SIZE
task_batch_size = self.calculate_batch_size(resources)

# Import in groups in order. Resources within groups are imported in parallel.
for package in packages:
Expand Down Expand Up @@ -427,6 +419,18 @@ def prepare_tasks(self, resource_types, packages, resources):
tasks.append(groups)
return tasks

def calculate_batch_size(self, resources):
# Count all items to determine batch size
all_count = 0
for _, item in resources.items():
for _, count in item.items():
all_count += count
if all_count > 50000:
task_batch_size = all_count / 1000
else:
task_batch_size = self.MIN_BATCH_SIZE
return task_batch_size

def schedule_tasks(self, tasks):
chained_tasks = chain()
for task in tasks:
Expand Down

0 comments on commit 93a51d7

Please sign in to comment.