Make pub year columns strings

sul-dlss · Jul 18, 2024 · df8a828 · df8a828
1 parent bfad4e8
commit df8a828
Showing 1 changed file with 3 additions and 3 deletions.
diff --git a/rialto_airflow/harvest/merge_pubs.py b/rialto_airflow/harvest/merge_pubs.py
@@ -46,7 +46,7 @@ def dimensions_pubs_df(dimensions_pubs):
     # Create a LazyFrame of dimension pubs to avoid loading all data into memory
     """
     # Polars is inferring volume is an integer, but it should be a string e.g. "97-B"
-    df = pl.scan_csv(dimensions_pubs, schema_overrides={"volume": pl.String})
+    df = pl.scan_csv(dimensions_pubs, schema_overrides={"volume": pl.String, "year": pl.String})
     df = df.select(
         pl.col("doi").map_elements(normalize_doi, return_dtype=pl.String),
         pl.col(
@@ -71,7 +71,7 @@ def openalex_pubs_df(openalex_pubs):
     """
     Create an openalex pubs LazyFrame and rename columns
     """
-    df = pl.scan_csv(openalex_pubs)
+    df = pl.scan_csv(openalex_pubs, schema_overrides={"publication_year": pl.String})
     df = df.select(
         pl.col("doi").map_elements(normalize_doi, return_dtype=pl.String),
         pl.col(
@@ -86,7 +86,7 @@ def sulpub_df(sul_pub):
     """
     Create a sulpub LazyFrame and rename columns
     """
-    df = pl.scan_csv(sul_pub, null_values="n/ a")
+    df = pl.scan_csv(sul_pub, schema_overrides={"year": pl.String})
     df = df.drop_nulls("doi")
     df = df.with_columns(
         pl.col("doi").map_elements(normalize_doi, return_dtype=pl.String)