Refactor format_citations and add legacy DOI

.gitignore

@@ -14,3 +14,6 @@ __pycache__/
 *.json
 *.pdf
 *.md
+
+!template.md
+!legacy.json

fetch_citations.py

@@ -30,8 +30,13 @@ def main():
 
     args = parser.parse_args()
 
+    # fetch DOI from data.isimip.org
     resources = fetch_resources(resources_url)
 
+    # load old DOI from json file
+    with open('legacy.json') as fp:
+        resources += json.load(fp)
+
     output_resources = []
     for resource in resources:
         if (args.doi and resource['doi'] not in args.doi) or (not args.all and resource.get('new_version')):

format_citations.py

@@ -1,6 +1,7 @@
 #!/usr/bin/env python3
 import argparse
 import json
+from datetime import datetime
 from pathlib import Path
 
 import pypandoc
@@ -9,6 +10,17 @@
 
 logger = structlog.get_logger()
 
+markdown_template = open('template.md').read()
+
+pandoc_template = '''
+---
+geometry: margin=1in
+colored-links: true
+linkcolor: blue
+urlcolor: orange
+...
+'''
+
 def main():
     parser = argparse.ArgumentParser()
 
@@ -20,11 +32,38 @@ def main():
     with open(args.input_file) as fp:
         resources = json.load(fp)
 
+    # sort resources by citation count
     resources = sorted([
         resource for resource in resources if resource['citations_count'] > 0
     ], key=lambda r: r['citations_count'], reverse=True)
 
-    rendered_template = Template(markdown_template).render(resources=resources)
+    # compile a list of publications
+    dois = set()
+    publications = {}
+    for resource in resources:
+        for resource in resources:
+            for citation in resource['citations']:
+                doi = citation.get('doi')
+
+                if doi not in dois:
+                    publication_year = citation.get('publication_year')
+                    if publication_year in publications:
+                        publications[publication_year].append(citation)
+                    else:
+                        publications[publication_year] = [citation]
+
+                dois.add(doi)
+
+    # sort publications in each year by date
+    for year in publications:
+        publications[year] = sorted(publications[year], key=lambda r: r['publication_date'], reverse=True)
+
+    # render the markdown template
+    rendered_template = Template(markdown_template).render(
+        resources=resources,
+        publications=publications,
+        today=datetime.today().strftime("%B %d, %Y")
+    )
 
     if args.output_path.suffix == '.md':
         args.output_path.write_text(rendered_template)
@@ -37,27 +76,5 @@ def main():
             extra_args=['--pdf-engine=xelatex']
         )
 
-markdown_template = '''# ISIMIP data citations
-{% for resource in resources %}
-## {{ resource.title_with_version }}
-
-DOI: <{{ resource.doi_url }}>
-
-Citations: {{ resource.citations_count }}
-
-{% for citation in resource.citations %}* {{ citation.creators_str }} ({{ citation.publication_year }}): **{{ citation.title }}**. {% if citation.journal %}{{ citation.journal }}{% else %}{{ citation.publisher }}{% endif %}. <{{ citation.doi_url}}>
-{% endfor %}
-{% endfor %}
-'''  # noqa: E501
-
-pandoc_template = '''
----
-geometry: margin=1in
-colored-links: true
-linkcolor: blue
-urlcolor: orange
-...
-'''
-
 if __name__ == '__main__':
     main()

legacy.json

@@ -0,0 +1,134 @@
+[
+  {
+    "doi": "10.5880/PIK.2016.001",
+    "doi_url": "https://doi.org/10.5880/PIK.2016.001",
+    "title": "Bias corrected GCM input data for ISIMIP Fast Track",
+    "title_with_version": "Bias corrected GCM input data for ISIMIP Fast Track",
+    "creators_str": "Hempel, Sabrina; Frieler, Katja; Warszawski, Lila; Schewe, Jacob; Piontek, Franziska",
+    "publication_year": "2013",
+    "publication_date": "2013",
+    "publisher": "GFZ Data Services",
+    "citation": "Hempel, Sabrina; Frieler, Katja; Warszawski, Lila; Schewe, Jacob; Piontek, Franziska (2013): Bias corrected GCM input data for ISIMIP Fast Track. GFZ Data Services. https://doi.org/10.5880/PIK.2016.001"
+  },
+  {
+    "doi": "10.5880/pik.2016.004",
+    "doi_url": "https://doi.org/10.5880/pik.2016.004",
+    "title": "EartH2Observe, WFDEI and ERA-Interim data Merged and Bias-corrected for ISIMIP (EWEMBI)",
+    "title_with_version": "EartH2Observe, WFDEI and ERA-Interim data Merged and Bias-corrected for ISIMIP (EWEMBI)",
+    "creators_str": "Lange, Stefan",
+    "publication_year": "2016",
+    "publication_date": "2016",
+    "publisher": "GFZ Data Services",
+    "citation": "Lange, Stefan (2016): EartH2Observe, WFDEI and ERA-Interim data Merged and Bias-corrected for ISIMIP (EWEMBI). GFZ Data Services. https://doi.org/10.5880/pik.2016.004"
+  },
+  {
+    "doi": "10.5880/PIK.2019.007",
+    "doi_url": "https://doi.org/10.5880/PIK.2019.007",
+    "title": "ISIMIP2a Simulation Data from Permafrost Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Permafrost Sector (V 1.1)",
+    "creators_str": "Arzhanov, Maxim; Betts, Richard; Eliseev, Alexey; Morfopoulos, Catherine; Schaphoff, Sibyll; Büchner, Matthias",
+    "publication_year": "2019",
+    "publication_date": "2019",
+    "publisher": "GFZ Data Services",
+    "citation": "Arzhanov, Maxim; Betts, Richard; Eliseev, Alexey; Morfopoulos, Catherine; Schaphoff, Sibyll; Büchner, Matthias (2019): ISIMIP2a Simulation Data from Permafrost Sector (V 1.1). GFZ Data Services. https://doi.org/10.5880/PIK.2019.007"
+  },
+  {
+    "doi": "10.5880/PIK.2019.005",
+    "doi_url": "https://doi.org/10.5880/PIK.2019.005",
+    "title": "ISIMIP2a Simulation Data from Biomes Sector (V. 1.1)",
+    "title_with_version": "ISIMIP2a Simulation Data from Biomes Sector",
+    "creators_str": "Reyer, Christopher; Asrar, Gassem; Betts, Richard; Chang, Jinfeng; Chen, Min; Ciais, Philippe; Dury, Marie; Francois, Louis; Henrot, Alexandra; Hickler, Thomas; Ito, Akihiko; Jacquemin, Ingrid; Nishina, Kazuya; Mishurov, Mikhail; Morfopoulos, Catherine; Munhoven, Guy; Ostberg, Sebastian; Pan, Shufen; Rafique, Rashid; Schaphoff, Sibyll; Steinkamp, Jörg; Tian, Hanqin; Ren, Wei; Yang, Jia; Zeng, Ning; Zhao, Fang; Büchner, Matthias",
+    "publication_year": "2019",
+    "publication_date": "2019",
+    "publisher": "GFZ Data Services",
+    "citation": "Reyer, Christopher; Asrar, Gassem; Betts, Richard; Chang, Jinfeng; Chen, Min; Ciais, Philippe; Dury, Marie; Francois, Louis; Henrot, Alexandra; Hickler, Thomas; Ito, Akihiko; Jacquemin, Ingrid; Nishina, Kazuya; Mishurov, Mikhail; Morfopoulos, Catherine; Munhoven, Guy; Ostberg, Sebastian; Pan, Shufen; Rafique, Rashid; Schaphoff, Sibyll; Steinkamp, Jörg; Tian, Hanqin; Ren, Wei; Yang, Jia; Zeng, Ning; Zhao, Fang; Büchner, Matthias (2019): ISIMIP2a Simulation Data from Biomes Sector (V. 1.1). GFZ Data Services. https://doi.org/10.5880/PIK.2019.005"
+  },
+  {
+    "doi": "10.5880/PIK.2019.012",
+    "doi_url": "https://doi.org/10.5880/PIK.2019.012",
+    "title": "ISIMIP2b Simulation Data from Biomes Sector",
+    "title_with_version": "ISIMIP2b Simulation Data from Biomes Sector",
+    "creators_str": "Reyer, Christopher; Chang, Jinfeng; Chen, Min; Forrest, Matt; François, Louis; Henrot, Alexandra-Jane; Hickler, Thomas; Ito, Akihito; Nishina, Kazuya; Ostberg, Sebastian; Schaphoff, Sibyll; Seneviratne, Sonia I.; Shi, Hao; Steinkamp, Jörg; Thiery, Wim; Tian, Hanqin; Zhao, Fang; Büchner, Matthias; Ciais, Philippe",
+    "publication_year": "2019",
+    "publication_date": "2019",
+    "publisher": "GFZ Data Services",
+    "citation": "Reyer, Christopher; Chang, Jinfeng; Chen, Min; Forrest, Matt; François, Louis; Henrot, Alexandra-Jane; Hickler, Thomas; Ito, Akihito; Nishina, Kazuya; Ostberg, Sebastian; Schaphoff, Sibyll; Seneviratne, Sonia I.; Shi, Hao; Steinkamp, Jörg; Thiery, Wim; Tian, Hanqin; Zhao, Fang; Büchner, Matthias; Ciais, Philippe (2019): ISIMIP2b Simulation Data from Biomes Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2019.012"
+  },
+  {
+    "doi": "10.5880/PIK.2019.003",
+    "doi_url": "https://doi.org/10.5880/PIK.2019.003",
+    "title": "ISIMIP2a Simulation Data from Water (global) Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Water (global) Sector (V. 1.1)",
+    "creators_str": "Gosling, Simon; Müller Schmied, Hannes; Betts, Richard; Chang, Jinfeng; Ciais, Philippe; Dankers, Rutger; Döll, Petra; Eisner, Stephanie; Flörke, Martina; Gerten, Dieter; Grillakis, Manolis; Hanasaki, Naota; Hagemann, Stefan; Huang, Maoyi; Huang, Zhongwei; Jerez, Sonia; Kim, Hyungjun; Koutroulis, Aristeidis; Leng, Guoyong; Liu, Xingcai; Masaki, Yoshimitsu; Montavez, Pedro; Morfopoulos, Catherine; Oki, Taikan; Papadimitriou, Lamprini; Pokhrel, Yadu; Portmann, Felix T.; Orth, Rene; Ostberg, Sebastian; Satoh, Yusuke; Seneviratne, Sonia; Sommer, Philipp; Stacke, Tobias; Tang, Qiuhong; Tsanis, Ioannis; Wada, Yoshihide; Zhou, Tian; Büchner, Matthias; Schewe, Jacob; Zhao, Fang",
+    "publication_year": "2019",
+    "publication_date": "2019",
+    "publisher": "GFZ Data Services",
+    "citation": "Gosling, Simon; Müller Schmied, Hannes; Betts, Richard; Chang, Jinfeng; Ciais, Philippe; Dankers, Rutger; Döll, Petra; Eisner, Stephanie; Flörke, Martina; Gerten, Dieter; Grillakis, Manolis; Hanasaki, Naota; Hagemann, Stefan; Huang, Maoyi; Huang, Zhongwei; Jerez, Sonia; Kim, Hyungjun; Koutroulis, Aristeidis; Leng, Guoyong; Liu, Xingcai; Masaki, Yoshimitsu; Montavez, Pedro; Morfopoulos, Catherine; Oki, Taikan; Papadimitriou, Lamprini; Pokhrel, Yadu; Portmann, Felix T.; Orth, Rene; Ostberg, Sebastian; Satoh, Yusuke; Seneviratne, Sonia; Sommer, Philipp; Stacke, Tobias; Tang, Qiuhong; Tsanis, Ioannis; Wada, Yoshihide; Zhou, Tian; Büchner, Matthias; Schewe, Jacob; Zhao, Fang (2019): ISIMIP2a Simulation Data from Water (global) Sector (V. 1.1). GFZ Data Services. https://doi.org/10.5880/PIK.2019.003"
+  },
+  {
+    "doi": "10.5880/PIK.2017.002",
+    "doi_url": "https://doi.org/10.5880/PIK.2017.002",
+    "title": "ISIMIP2a Simulation Data from Biomes Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Biomes Sector",
+    "creators_str": "Reyer, Christopher; Asrar, Gassem; Betts, Richard; Chang, Jinfeng; Chen, Min; Ciais, Philippe; Dury, Marie; Francois, Louis; Henrot, Alexandra; Hickler, Thomas; Ito, Akihiko; Jacquemin, Ingrid; Nishina, Kazuya; Mishurov, Mikhail; Morfopoulos, Catherine; Munhoven, Guy; Ostberg, Sebastian; Pan, Shufen; Rafique, Rashid; Schaphoff, Sibyll; Steinkamp, Jörg; Tian, Hanqin; Ren, Wei; Yang, Jia; Zeng, Ning; Zhao, Fang; Büchner, Matthias",
+    "publication_year": "2017",
+    "publication_date": "2017",
+    "publisher": "GFZ Data Services",
+    "citation": "Reyer, Christopher; Asrar, Gassem; Betts, Richard; Chang, Jinfeng; Chen, Min; Ciais, Philippe; Dury, Marie; Francois, Louis; Henrot, Alexandra; Hickler, Thomas; Ito, Akihiko; Jacquemin, Ingrid; Nishina, Kazuya; Mishurov, Mikhail; Morfopoulos, Catherine; Munhoven, Guy; Ostberg, Sebastian; Pan, Shufen; Rafique, Rashid; Schaphoff, Sibyll; Steinkamp, Jörg; Tian, Hanqin; Ren, Wei; Yang, Jia; Zeng, Ning; Zhao, Fang; Büchner, Matthias (2017): ISIMIP2a Simulation Data from Biomes Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2017.002"
+  },
+  {
+    "doi": "10.5880/PIK.2018.006",
+    "doi_url": "https://doi.org/10.5880/PIK.2018.006",
+    "title": "ISIMIP2a Simulation Data from Permafrost Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Permafrost Sector",
+    "creators_str": "Arzhanov, Maxim; Betts, Richard; Eliseev, Alexey; Morfopoulos, Catherine; Schaphoff, Sibyll; Büchner, Matthias",
+    "publication_year": "2018",
+    "publication_date": "2018",
+    "publisher": "GFZ Data Services",
+    "citation": "Arzhanov, Maxim; Betts, Richard; Eliseev, Alexey; Morfopoulos, Catherine; Schaphoff, Sibyll; Büchner, Matthias (2018): ISIMIP2a Simulation Data from Permafrost Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2018.006"
+  },
+  {
+    "doi": "10.5880/PIK.2017.006",
+    "doi_url": "https://doi.org/10.5880/PIK.2017.006",
+    "title": "ISIMIP2a Simulation Data from Agricultural Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Agricultural Sector",
+    "creators_str": "Arneth, Almut; Balkovic, Juraj; Ciais, Philippe; de Wit, Allard; Deryng, Delphine; Elliott, Joshua; Folberth, Christian; Glotter, Michael; Iizumi, Toshichika; Izaurralde, Roberto C.; Jones, Andrew D.; Khabarov, Nikolay; Lawrence, Peter; Liu, Wenfeng; Mitter, Hermine; Müller, Christoph; Olin, Stefan; Pugh, Thomas A. M.; Reddy, Ashwan D.; Sakurai, Gen; Schmid, Erwin; Wang, Xuhui; Wu, Xiuchen; Yang, Hong; Büchner, Matthias",
+    "publication_year": "2017",
+    "publication_date": "2017",
+    "publisher": "GFZ Data Services",
+    "citation": "Arneth, Almut; Balkovic, Juraj; Ciais, Philippe; de Wit, Allard; Deryng, Delphine; Elliott, Joshua; Folberth, Christian; Glotter, Michael; Iizumi, Toshichika; Izaurralde, Roberto C.; Jones, Andrew D.; Khabarov, Nikolay; Lawrence, Peter; Liu, Wenfeng; Mitter, Hermine; Müller, Christoph; Olin, Stefan; Pugh, Thomas A. M.; Reddy, Ashwan D.; Sakurai, Gen; Schmid, Erwin; Wang, Xuhui; Wu, Xiuchen; Yang, Hong; Büchner, Matthias (2017): ISIMIP2a Simulation Data from Agricultural Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2017.006"
+  },
+  {
+    "doi": "10.5880/PIK.2017.010",
+    "doi_url": "https://doi.org/10.5880/PIK.2017.010",
+    "title": "ISIMIP2a Simulation Data from Water (global) Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Water (global) Sector",
+    "creators_str": "Gosling, Simon; Müller Schmied, Hannes; Betts, Richard; Chang, Jinfeng; Ciais, Philippe; Dankers, Rutger; Döll, Petra; Eisner, Stephanie; Flörke, Martina; Gerten, Dieter; Grillakis, Manolis; Hanasaki, Naota; Hagemann, Stefan; Huang, Maoyi; Huang, Zhongwei; Jerez, Sonia; Kim, Hyungjun; Koutroulis, Aristeidis; Leng, Guoyong; Liu, Xingcai; Masaki, Yoshimitsu; Montavez, Pedro; Morfopoulos, Catherine; Oki, Taikan; Papadimitriou, Lamprini; Pokhrel, Yadu; Portmann, Felix T.; Orth, Rene; Ostberg, Sebastian; Satoh, Yusuke; Seneviratne, Sonia; Sommer, Philipp; Stacke, Tobias; Tang, Qiuhong; Tsanis, Ioannis; Wada, Yoshihide; Zhou, Tian; Büchner, Matthias; Schewe, Jacob; Zhao, Fang",
+    "publication_year": "2017",
+    "publication_date": "2017",
+    "publisher": "GFZ Data Services",
+    "citation": "Gosling, Simon; Müller Schmied, Hannes; Betts, Richard; Chang, Jinfeng; Ciais, Philippe; Dankers, Rutger; Döll, Petra; Eisner, Stephanie; Flörke, Martina; Gerten, Dieter; Grillakis, Manolis; Hanasaki, Naota; Hagemann, Stefan; Huang, Maoyi; Huang, Zhongwei; Jerez, Sonia; Kim, Hyungjun; Koutroulis, Aristeidis; Leng, Guoyong; Liu, Xingcai; Masaki, Yoshimitsu; Montavez, Pedro; Morfopoulos, Catherine; Oki, Taikan; Papadimitriou, Lamprini; Pokhrel, Yadu; Portmann, Felix T.; Orth, Rene; Ostberg, Sebastian; Satoh, Yusuke; Seneviratne, Sonia; Sommer, Philipp; Stacke, Tobias; Tang, Qiuhong; Tsanis, Ioannis; Wada, Yoshihide; Zhou, Tian; Büchner, Matthias; Schewe, Jacob; Zhao, Fang (2017): ISIMIP2a Simulation Data from Water (global) Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2017.010"
+  },
+  {
+    "doi": "10.5880/PIK.2018.004",
+    "doi_url": "https://doi.org/10.5880/PIK.2018.004",
+    "title": "ISIMIP2a Simulation Data from Fisheries & Marine Ecosystems (Fish-MIP; regional) Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Fisheries & Marine Ecosystems (Fish-MIP; regional) Sector",
+    "creators_str": "Eddy, Tyler D.; Bulman, Cathy M.; Cheung, William W.L.; Coll, Marta; Fulton, Elizabeth A.; Galbraith, Eric D.; Lotze, Heike K.; Mackinson, Steve; Niiranen, Susa; Oliveros-Ramos, Ricardo; Piroddi, Chiara; Steenbeek, Jeroen; Tittensor, Derek P.; Walker, Nicola D.; Schewe, Jacob; Volkholz, Jan; Büchner, Matthias",
+    "publication_year": "2018",
+    "publication_date": "2018",
+    "publisher": "GFZ Data Services",
+    "citation": "Eddy, Tyler D.; Bulman, Cathy M.; Cheung, William W.L.; Coll, Marta; Fulton, Elizabeth A.; Galbraith, Eric D.; Lotze, Heike K.; Mackinson, Steve; Niiranen, Susa; Oliveros-Ramos, Ricardo; Piroddi, Chiara; Steenbeek, Jeroen; Tittensor, Derek P.; Walker, Nicola D.; Schewe, Jacob; Volkholz, Jan; Büchner, Matthias (2018): ISIMIP2a Simulation Data from Fisheries & Marine Ecosystems (Fish-MIP; regional) Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2018.004"
+  },
+  {
+    "doi": "10.5880/PIK.2018.005",
+    "doi_url": "https://doi.org/10.5880/PIK.2018.005",
+    "title": "ISIMIP2a Simulation Data from Fisheries & Marine Ecosystems (Fish-MIP; global) Sector",
+    "title_with_version": "ISIMIP2a Simulation Data from Fisheries & Marine Ecosystems (Fish-MIP; global) Sector",
+    "creators_str": "Tittensor, Derek P.; Lotze, Heike K.; Eddy, Tyler D.; Galbraith, Eric D.; Cheung, William W.L.; Bryndum-Buchholz, Andrea; Barange, Manuel; Barrier, Nicolas; Bianchi, Daniele; Blanchard, Julia L.; Bopp, Laurent; Carozza, David A.; Christensen, Villy; Coll, Marta; Jennings, Simon; Jones, Miranda; Maury, Olivier; Silva, Tiago A.M.; Steenbeek, Jeroen; Verley, Philippe; Schewe, Jacob; Volkholz, Jan; Büchner, Matthias",
+    "publication_year": "2018",
+    "publication_date": "2018",
+    "publisher": "GFZ Data Services",
+    "citation": "Tittensor, Derek P.; Lotze, Heike K.; Eddy, Tyler D.; Galbraith, Eric D.; Cheung, William W.L.; Bryndum-Buchholz, Andrea; Barange, Manuel; Barrier, Nicolas; Bianchi, Daniele; Blanchard, Julia L.; Bopp, Laurent; Carozza, David A.; Christensen, Villy; Coll, Marta; Jennings, Simon; Jones, Miranda; Maury, Olivier; Silva, Tiago A.M.; Steenbeek, Jeroen; Verley, Philippe; Schewe, Jacob; Volkholz, Jan; Büchner, Matthias (2018): ISIMIP2a Simulation Data from Fisheries & Marine Ecosystems (Fish-MIP; global) Sector. GFZ Data Services. https://doi.org/10.5880/PIK.2018.005"
+  }
+]

template.md

@@ -0,0 +1,29 @@
+# ISIMIP data citations
+
+{{ today }}
+
+| Dataset                                                           | No. of Citations |
+| ----------------------------------------------------------------- | ---------------- |{% for resource in resources %}
+| {{ resource.title_with_version }} | {{ resource.citations_count }} |{% endfor %}
+
+## Dataset citations
+
+{% for resource in resources %}
+### {{ resource.title_with_version }}
+
+DOI: <{{ resource.doi_url }}>
+
+Citations: {{ resource.citations_count }}
+
+{% for citation in resource.citations %}* {{ citation.creators_str }} ({{ citation.publication_year }}): **{{ citation.title }}**. {% if citation.journal %}{{ citation.journal }}{% else %}{{ citation.publisher }}{% endif %}. <{{ citation.doi_url}}>
+{% endfor %}
+{% endfor %}
+
+
+## Citing publications by year
+
+{% for year, publication_list in publications.items() %}
+### {{ year }}
+
+{% for publication in publication_list %}* {{ publication.creators_str }} ({{ publication.publication_year }}): **{{ publication.title }}**. {% if publication.journal %}{{ publication.journal }}{% else %}{{ publication.publisher }}{% endif %}. <{{ publication.doi_url}}>.
+{% endfor %}{% endfor %}