This groundwork enables ecosystem players to consider deploying RAG solutions in real time without having to configure data retrieval systems.
Link to Louis Brulé-Naudet's Hugging Face profile
```python import concurrent.futures import logging
from datasets from tqdm import tqdm
def dataset_loader( name:str, streaming:bool=True ) -> datasets.Dataset: """ Helper function to load a single dataset in parallel.
Parameters ---------- name : str Name of the dataset to be loaded. streaming : bool, optional Determines if datasets are streamed. Default is True. Returns ------- dataset : datasets.Dataset Loaded dataset object. Raises ------ Exception If an error occurs during dataset loading. """ try: return datasets.load_dataset( name, split="train", streaming=streaming ) except Exception as exc: logging.error(f"Error loading dataset {name}: {exc}") return None
def load_datasets( req:list, streaming:bool=True ) -> list: """ Downloads datasets specified in a list and creates a list of loaded datasets.
Parameters ---------- req : list A list containing the names of datasets to be downloaded. streaming : bool, optional Determines if datasets are streamed. Default is True. Returns ------- datasets_list : list A list containing loaded datasets as per the requested names provided in 'req'. Raises ------ Exception If an error occurs during dataset loading or processing. Examples -------- >>> datasets = load_datasets(["dataset1", "dataset2"], streaming=False) """ datasets_list = [] with concurrent.futures.ThreadPoolExecutor() as executor: future_to_dataset = {executor.submit(dataset_loader, name): name for name in req} for future in tqdm(concurrent.futures.as_completed(future_to_dataset), total=len(req)): name = future_to_dataset[future] try: dataset = future.result() if dataset: datasets_list.append(dataset) except Exception as exc: logging.error(f"Error processing dataset {name}: {exc}") return datasets_list
req = [ "louisbrulenaudet/code-artisanat", "louisbrulenaudet/code-action-sociale-familles", "louisbrulenaudet/code-assurances", "louisbrulenaudet/code-aviation-civile", "louisbrulenaudet/code-cinema-image-animee", "louisbrulenaudet/code-civil", "louisbrulenaudet/code-commande-publique", "louisbrulenaudet/code-commerce", "louisbrulenaudet/code-communes", "louisbrulenaudet/code-communes-nouvelle-caledonie", "louisbrulenaudet/code-consommation", "louisbrulenaudet/code-construction-habitation", "louisbrulenaudet/code-defense", "louisbrulenaudet/code-deontologie-architectes", "louisbrulenaudet/code-disciplinaire-penal-marine-marchande", "louisbrulenaudet/code-domaine-etat", "louisbrulenaudet/code-domaine-etat-collectivites-mayotte", "louisbrulenaudet/code-domaine-public-fluvial-navigation-interieure", "louisbrulenaudet/code-douanes", "louisbrulenaudet/code-douanes-mayotte", "louisbrulenaudet/code-education", "louisbrulenaudet/code-electoral", "louisbrulenaudet/code-energie", "louisbrulenaudet/code-entree-sejour-etrangers-droit-asile", "louisbrulenaudet/code-environnement", "louisbrulenaudet/code-expropriation-utilite-publique", "louisbrulenaudet/code-famille-aide-sociale", "louisbrulenaudet/code-forestier-nouveau", "louisbrulenaudet/code-fonction-publique", "louisbrulenaudet/code-propriete-personnes-publiques", "louisbrulenaudet/code-collectivites-territoriales", "louisbrulenaudet/code-impots", "louisbrulenaudet/code-impots-annexe-i", "louisbrulenaudet/code-impots-annexe-ii", "louisbrulenaudet/code-impots-annexe-iii", "louisbrulenaudet/code-impots-annexe-iv", "louisbrulenaudet/code-impositions-biens-services", "louisbrulenaudet/code-instruments-monetaires-medailles", "louisbrulenaudet/code-juridictions-financieres", "louisbrulenaudet/code-justice-administrative", "louisbrulenaudet/code-justice-militaire-nouveau", "louisbrulenaudet/code-justice-penale-mineurs", "louisbrulenaudet/code-legion-honneur-medaille-militaire-ordre-national-merite", "louisbrulenaudet/livre-procedures-fiscales", "louisbrulenaudet/code-minier", "louisbrulenaudet/code-minier-nouveau", "louisbrulenaudet/code-monetaire-financier", "louisbrulenaudet/code-mutualite", "louisbrulenaudet/code-organisation-judiciaire", "louisbrulenaudet/code-patrimoine", "louisbrulenaudet/code-penal", "louisbrulenaudet/code-penitentiaire", "louisbrulenaudet/code-pensions-civiles-militaires-retraite", "louisbrulenaudet/code-pensions-retraite-marins-francais-commerce-peche-plaisance", "louisbrulenaudet/code-pensions-militaires-invalidite-victimes-guerre", "louisbrulenaudet/code-ports-maritimes", "louisbrulenaudet/code-postes-communications-electroniques", "louisbrulenaudet/code-procedure-civile", "louisbrulenaudet/code-procedure-penale", "louisbrulenaudet/code-procedures-civiles-execution", "louisbrulenaudet/code-propriete-intellectuelle", "louisbrulenaudet/code-recherche", "louisbrulenaudet/code-relations-public-administration", "louisbrulenaudet/code-route", "louisbrulenaudet/code-rural-ancien", "louisbrulenaudet/code-rural-peche-maritime", "louisbrulenaudet/code-sante-publique", "louisbrulenaudet/code-securite-interieure", "louisbrulenaudet/code-securite-sociale", "louisbrulenaudet/code-service-national", "louisbrulenaudet/code-sport", "louisbrulenaudet/code-tourisme", "louisbrulenaudet/code-transports", "louisbrulenaudet/code-travail", "louisbrulenaudet/code-travail-maritime", "louisbrulenaudet/code-urbanisme", "louisbrulenaudet/code-voirie-routiere" ]
dataset = load_datasets( req=req, streaming=True
) ```