Add sampling parameters to env

.env-Sample

@@ -26,4 +26,7 @@ RUN_CONFIG_IS_ASYNC="true"
 NEO4J_URI=bolt://localhost:7687## use bolt://neo4j:7687 if using docker for ragbuilder
 NEO4J_USERNAME=neo4j
 NEO4J_PASSWORD=ragbuilder
-NEO4J_LOAD=true # set to false if graph is already loaded and you don't want to reload
+NEO4J_LOAD=true # set to false if graph is already loaded and you don't want to reload
+SAMPLING_RATIO=0.10 # Sampling ratio: If set to 0.10, ~10% of original data will be sampled, and used for RAG building.
+SAMPLING_SIZE_THRESHOLD=750_000 # If your source data is larger than this threshold, RAGBuilder will default to sampling.
+SAMPLING_FILE_SIZE_THRESHOLD=500_000 # When sampling directories, individual files that are larger this threshold, will be sampled at file level.

src/ragbuilder/sampler.py

@@ -15,15 +15,19 @@
 
 setup_logging()
 logger = logging.getLogger("ragbuilder")
+SAMPLING_RATIO = float(os.getenv('SAMPLING_RATIO', '0.1'))
+SAMPLING_SIZE_THRESHOLD = int(os.getenv('SAMPLING_SIZE_THRESHOLD', '750_000'))
+SAMPLING_FILE_SIZE_THRESHOLD = int(os.getenv('SAMPLING_FILE_SIZE_THRESHOLD', '500_000'))
+
 
 class DataSampler:
     def __init__(
             self, 
             data_source: str, 
             enable_sampling: bool = True, 
-            sample_size_threshold: int =  750_000, #1_000_000_000, 
-            sample_ratio: float = 0.1, 
-            file_size_threshold: int = 500_000 #1_000_000
+            sample_size_threshold: int =  SAMPLING_SIZE_THRESHOLD,
+            sample_ratio: float = SAMPLING_RATIO, 
+            file_size_threshold: int = SAMPLING_FILE_SIZE_THRESHOLD
     ):
         self.data_source = data_source
         self.enable_sampling = enable_sampling