diff --git a/install_usermodule_r8.sh b/install_usermodule_r8.sh index f8588ef2..6f8b8ccb 100755 --- a/install_usermodule_r8.sh +++ b/install_usermodule_r8.sh @@ -52,11 +52,9 @@ install_snappy() { ) ( - export VERSION="$SNAPPY_VERSION" # Pip install messes up shebang: https://github.com/pypa/setuptools/issues/494 # pip install ./utils/SnapPy/ - cd utils/SnapPy || exit 2 - python3 setup.py install + pip install ./utils/SnapPy ) } @@ -70,14 +68,8 @@ install_bsnap() { main() { if [ -z "$1" ]; then MODULE_VERSION="TEST" - SNAPPY_VERSION="0.0.0.dev0" else MODULE_VERSION="$1" - if [ -z "$2" ]; then - SNAPPY_VERSION="$MODULE_VERSION" - else - SNAPPY_VERSION="$2" - fi fi MODULE_PREFIX=/modules/rhel8/user-apps/fou-modules/SnapPy/"$MODULE_VERSION"/ diff --git a/src/naccident/Makefile b/src/naccident/Makefile index acc6fda4..1eac9789 100644 --- a/src/naccident/Makefile +++ b/src/naccident/Makefile @@ -15,9 +15,8 @@ clean: clean_links distclean: clean rm -f bsnap_naccident create_naccident_input create_naccident_output -install: bsnap_naccident ../../utils/SnapPy/Snappy/AddToa.py +install: bsnap_naccident install bsnap_naccident $(BINDIR)/ - install ../../utils/SnapPy/Snappy/AddToa.py $(BINDIR)/snapAddToa .PHONY: all clean distclean install diff --git a/utils/SnapPy/Snappy/AddToa.py b/utils/SnapPy/Snappy/AddToa.py index 8f5ae58f..cda305c0 100755 --- a/utils/SnapPy/Snappy/AddToa.py +++ b/utils/SnapPy/Snappy/AddToa.py @@ -94,7 +94,7 @@ def add_toa_to_nc(nc: netCDF4.Dataset): totalVar[:] = total -if __name__ == "__main__": +def main(): import argparse parser = argparse.ArgumentParser() @@ -103,3 +103,7 @@ def add_toa_to_nc(nc: netCDF4.Dataset): with netCDF4.Dataset(args.snapNc, "a") as nc: add_toa_to_nc(nc) + + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/debian.bionic/control b/utils/SnapPy/debian.bionic/control index 9860721f..c0e2e49c 100644 --- a/utils/SnapPy/debian.bionic/control +++ b/utils/SnapPy/debian.bionic/control @@ -2,13 +2,13 @@ Source: snap-py Maintainer: Heiko Klein Section: python Priority: optional -Build-Depends: python3-all, python3-netcdf4, debhelper (>= 7.4.3) +Build-Depends: python3-all, python3-setuptools, debhelper (>= 7.4.3) Standards-Version: 3.9.7 Package: snap-py Architecture: all -Depends: ${misc:Depends}, ${python3:Depends}, bsnap (>= 2.1.2), python3-pyqt5, python3-pyqt5.qtwebkit +Depends: ${misc:Depends}, ${python3:Depends}, bsnap (>= 2.1.2), python3-pyqt5, python3-pyqt5.qtwebkit, python3-netcdf4, python3-numpy Description: SNAP GUI in python diff --git a/utils/SnapPy/debian.bionic/rules b/utils/SnapPy/debian.bionic/rules index 355af278..158ca5fe 100755 --- a/utils/SnapPy/debian.bionic/rules +++ b/utils/SnapPy/debian.bionic/rules @@ -19,11 +19,11 @@ override_dh_auto_build: override_dh_auto_install: - python3 setup.py install --force --root=debian/snap-py --no-compile -O0 --install-layout=deb + python3 setup.py install --force --root=debian/snap-py --no-compile -O0 --install-layout=deb --prefix=/usr override_dh_python2: - echo "omit: dh_python2 --no-guessing-versions" + dh_python2 --no-guessing-versions diff --git a/utils/SnapPy/setup.cfg b/utils/SnapPy/setup.cfg new file mode 100644 index 00000000..c3a3d3c3 --- /dev/null +++ b/utils/SnapPy/setup.cfg @@ -0,0 +1,45 @@ +[metadata] +name = Snappy +version = 2.3.0 +description = SNAP utilities in python +url = https://gitlab.met.no/emep/snap +author = Heiko Klein +author_email = Heiko.Klein@met.no +license = GPL-3.0-only + +[options] +zip_safe = False +include_package_data = True +packages = find: +setup_requires = + setuptools +install_requires = + netCDF4 + numpy + +[options.package_data] +Snappy = + resources/* +Snappy.EEMEP = + resources/* + +[options.entry_points] +gui_scripts = + snapPy = snapscripts.snapPy:main +console_scripts = + snap4rimsterm = snapscripts.snap4rimsterm:main + snapAddToa = snappy.AddToa:main + snapCombineInverse = snapscripts.snapCombineInverse:main + snapNc2grib = snapscripts.snapNc2grib:main + snapRunnerNpps = snapscripts.snapRunnerNpps:main + snapRunnerNpp = snapscripts.snapRunnerNpp:main + eemepModelRunner = snapscripts.eemepModelRunner:main + snapRemoteRunner = snapscripts.snapRemoteRunner:main + snapEnsPlot = snapscripts.snapEnsPlot:main + + +[options.extras_require] +gui = + cartopy + # PyQt5 + # PyQtWebKit diff --git a/utils/SnapPy/setup.py b/utils/SnapPy/setup.py index 97304c4b..9ad0e80e 100644 --- a/utils/SnapPy/setup.py +++ b/utils/SnapPy/setup.py @@ -1,46 +1,4 @@ -#!/usr/bin/env python3 -# -# SNAP: Servere Nuclear Accident Programme -# Copyright (C) 1992-2017 Norwegian Meteorological Institute -# -# This file is part of SNAP. SNAP is free software: you can -# redistribute it and/or modify it under the terms of the -# GNU General Public License as published by the -# Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see . -# +#! /usr/bin/env python3 +from setuptools import setup -from distutils.core import setup -import os - -version = os.getenv("VERSION", "0.5") - -setup( - name="Snappy", - version=version, - description="SNAP GUI in python", - author="Heiko Klein", - author_email="Heiko.Klein@met.no", - url="https://gitlab.met.no/emep/snap", - packages=["Snappy", "Snappy.EEMEP", "METNO"], - package_dir={"Snappy": "Snappy", "Snappy.EEMEP": "Snappy/EEMEP"}, - package_data={"Snappy": ["resources/*"], "Snappy.EEMEP": ["resources/*"]}, - scripts=[ - "snapPy", - "snap4rimsterm", - "snapCombineInverse", - "snapNc2grib.py", - "snapRunnerNpps", - "snapRunnerNpp", - "eemepModelRunner", - "snapRemoteRunner.py", - ], -) +setup() diff --git a/utils/SnapPy/snapscripts/__init__.py b/utils/SnapPy/snapscripts/__init__.py new file mode 100644 index 00000000..e69de29b diff --git a/utils/SnapPy/eemepModelRunner b/utils/SnapPy/snapscripts/eemepModelRunner.py similarity index 99% rename from utils/SnapPy/eemepModelRunner rename to utils/SnapPy/snapscripts/eemepModelRunner.py index d8a3a2c2..c1b933ef 100755 --- a/utils/SnapPy/eemepModelRunner +++ b/utils/SnapPy/snapscripts/eemepModelRunner.py @@ -176,7 +176,7 @@ def __init__(self, directory, hpc, directory2, dryrun=False): -if __name__ == '__main__': +def main(): os.umask(0) import argparse parser = argparse.ArgumentParser(description="Search for volcano.xml/npp.xml files in the top-level directory, take the newest one and run the model on a HPC machine") @@ -197,3 +197,7 @@ def __init__(self, directory, hpc, directory2, dryrun=False): delete_oldfiles(args.dir, args.cleanup) if dir2 and dirIsWritable(dir2): delete_oldfiles(dir2, args.cleanup) + + +if __name__ == '__main__': + main() diff --git a/utils/SnapPy/snap4rimsterm b/utils/SnapPy/snapscripts/snap4rimsterm.py similarity index 99% rename from utils/SnapPy/snap4rimsterm rename to utils/SnapPy/snapscripts/snap4rimsterm.py index ebc2e4eb..6c92f1f7 100755 --- a/utils/SnapPy/snap4rimsterm +++ b/utils/SnapPy/snapscripts/snap4rimsterm.py @@ -245,7 +245,7 @@ def snap4rimsterm(rimsterm, argosrequest, basedir, ident, met_model, bitmapCompr -if __name__ == "__main__": +def main(): os.umask(0) # make sure files can be deleted later import argparse parser = argparse.ArgumentParser(description="run snap from a rimsterm.xml file and convert to grib-files named ident_conc, ident_depo ident_wetd ident_dose ident_tofa") @@ -261,3 +261,7 @@ def snap4rimsterm(rimsterm, argosrequest, basedir, ident, met_model, bitmapCompr snap4rimsterm(args.rimsterm, args.argosrequest, args.dir, args.ident, args.metmodel, bitmapCompress=args.bitmapCompress) else: print("need rimsterm option", file=sys.stderr) + + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/snapCombineInverse b/utils/SnapPy/snapscripts/snapCombineInverse.py similarity index 86% rename from utils/SnapPy/snapCombineInverse rename to utils/SnapPy/snapscripts/snapCombineInverse.py index 7573bd43..18311848 100755 --- a/utils/SnapPy/snapCombineInverse +++ b/utils/SnapPy/snapscripts/snapCombineInverse.py @@ -135,21 +135,25 @@ def initializeNc(ncFile, outNcFile, isotope): -parser = argparse.ArgumentParser() -parser.add_argument("-i", required=True, action="append", help="one or more snap.nc file from an inverse run to a measurement") -parser.add_argument("-n", action="append", help="zero or more snap.nc file from an inverse run to a stations without measurement") -parser.add_argument("-o", required=True, help="output nc file") -parser.add_argument("-I", default="I131", help="isotope (default: I131)") -args = parser.parse_args() - -(times, out_nc, conc, prob) = initializeNc(args.i[0], args.o, args.I) -# remove first input -args.i.pop(0) -for i in args.i: - (conc, prob) = addFile(i, args.I, times, conc, prob) - -if args.n: - for i in args.n: - (conc, prob) = addFile(i, args.I, times, conc, prob, False) - -finishOutput(out_nc, args.I, times, conc, prob) +def main(): + parser = argparse.ArgumentParser() + parser.add_argument("-i", required=True, action="append", help="one or more snap.nc file from an inverse run to a measurement") + parser.add_argument("-n", action="append", help="zero or more snap.nc file from an inverse run to a stations without measurement") + parser.add_argument("-o", required=True, help="output nc file") + parser.add_argument("-I", default="I131", help="isotope (default: I131)") + args = parser.parse_args() + + (times, out_nc, conc, prob) = initializeNc(args.i[0], args.o, args.I) + # remove first input + args.i.pop(0) + for i in args.i: + (conc, prob) = addFile(i, args.I, times, conc, prob) + + if args.n: + for i in args.n: + (conc, prob) = addFile(i, args.I, times, conc, prob, False) + + finishOutput(out_nc, args.I, times, conc, prob) + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/snapEnsPlot.py b/utils/SnapPy/snapscripts/snapEnsPlot.py similarity index 99% rename from utils/SnapPy/snapEnsPlot.py rename to utils/SnapPy/snapscripts/snapEnsPlot.py index 918bc4dc..eb619537 100644 --- a/utils/SnapPy/snapEnsPlot.py +++ b/utils/SnapPy/snapscripts/snapEnsPlot.py @@ -14,14 +14,16 @@ import sys import numpy as np +import logging # suppress some warnings warnings.filterwarnings("ignore", category=UserWarning, message="Warning: 'partition' will ignore the 'mask' of the MaskedArray.") # shapefile.py uses root logger :-( and warns a lot about GSHHS -import logging logging.root.setLevel(logging.ERROR) + + def plotMap(data, x, y, ax, title="", title_loc="center", clevs=[10,100,300,1000,3000,10000,30000,100000, 300000, 10000000], colors=None, extend='max'): ax.add_feature(cartopy.feature.GSHHSFeature(scale='low', facecolor='none', edgecolor='whitesmoke', linewidth=.2), zorder=100) ax.add_feature(cartopy.feature.BORDERS, edgecolor="lightgray", linewidth=.5, zorder=100) @@ -266,9 +268,8 @@ def snapens(ncfiles, hour, outfile): fig.subplots_adjust(hspace=0.12, wspace=0.01) fig.savefig(outfile, bbox_inches='tight') - -if __name__ == "__main__": +def main(): os.umask(0o002) parser = argparse.ArgumentParser( @@ -281,4 +282,8 @@ def snapens(ncfiles, hour, outfile): parser.add_argument('SNAPNC', help="snap*.nc filenames", nargs='+') args = parser.parse_args() - snapens(args.SNAPNC, args.hour, args.out) \ No newline at end of file + snapens(args.SNAPNC, args.hour, args.out) + + +if __name__ == "main": + main() diff --git a/utils/SnapPy/snapNc2grib.py b/utils/SnapPy/snapscripts/snapNc2grib.py similarity index 98% rename from utils/SnapPy/snapNc2grib.py rename to utils/SnapPy/snapscripts/snapNc2grib.py index fda90f3a..14164ac0 100755 --- a/utils/SnapPy/snapNc2grib.py +++ b/utils/SnapPy/snapscripts/snapNc2grib.py @@ -21,7 +21,7 @@ def getIsotopesFromFile(filename): print(f"converting isotopes: {', '.join(isotop_names)}", file=sys.stderr) return isotopes -if __name__ == "__main__": +def main(): parser = argparse.ArgumentParser(description="convert a snap.nc output-file to grib, should be run after snapAddToa") parser.add_argument("--nc", help="snap.nc filename", required=True) parser.add_argument("--ident", help="output-file identifier", required=True) @@ -37,3 +37,6 @@ def getIsotopesFromFile(filename): bitmapCompress= True dirname = os.path.dirname(ncfile) snapNc_convert_to_grib(ncfile, dirname, ident, isotopes, bitmapCompress=bitmapCompress) + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/snapPy b/utils/SnapPy/snapscripts/snapPy.py similarity index 98% rename from utils/SnapPy/snapPy rename to utils/SnapPy/snapscripts/snapPy.py index 4002fe7d..6b3715da 100755 --- a/utils/SnapPy/snapPy +++ b/utils/SnapPy/snapscripts/snapPy.py @@ -26,7 +26,7 @@ import Snappy.EEMEP.Resources import Snappy.EEMEP.Controller -if __name__ == "__main__": +def main(): app = QtWidgets.QApplication(sys.argv) tabs = QtWidgets.QTabWidget() snap = SnapController() @@ -42,3 +42,6 @@ sys.exit(app.exec_()) + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/snapRemoteRunner.py b/utils/SnapPy/snapscripts/snapRemoteRunner.py similarity index 99% rename from utils/SnapPy/snapRemoteRunner.py rename to utils/SnapPy/snapscripts/snapRemoteRunner.py index a5dc20bf..9247629a 100755 --- a/utils/SnapPy/snapRemoteRunner.py +++ b/utils/SnapPy/snapscripts/snapRemoteRunner.py @@ -403,7 +403,7 @@ def _check_and_unpack_new_files(self): self.ssh.syscall("rm", delete_upload_files, 30) -if __name__ == "__main__": +def main(): os.umask(0o002) import argparse @@ -482,3 +482,7 @@ def _check_and_unpack_new_files(self): delete_oldfiles(args.dir, args.cleanup) if dir2 and dirIsWritable(dir2): delete_oldfiles(dir2, args.cleanup) + + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/snapRunnerNpp b/utils/SnapPy/snapscripts/snapRunnerNpp.py similarity index 99% rename from utils/SnapPy/snapRunnerNpp rename to utils/SnapPy/snapscripts/snapRunnerNpp.py index fdde0bb0..473d2d5b 100755 --- a/utils/SnapPy/snapRunnerNpp +++ b/utils/SnapPy/snapscripts/snapRunnerNpp.py @@ -345,7 +345,7 @@ def runsnap(npp, met_model, outdir, release_dt, runtime, plot_from_file=None): -if __name__ == "__main__": +def main(): os.umask(0o002) parser = argparse.ArgumentParser( @@ -373,3 +373,7 @@ def runsnap(npp, met_model, outdir, release_dt, runtime, plot_from_file=None): if (args.store): os.environ["STORE"] = args.store runsnap(outdir=args.outdir, met_model=args.metmodel, runtime=int(args.runtime), npp=args.npp, release_dt=release_dt, plot_from_file=plot_from_file) + + +if __name__ == "__main__": + main() diff --git a/utils/SnapPy/snapRunnerNpps b/utils/SnapPy/snapscripts/snapRunnerNpps.py similarity index 79% rename from utils/SnapPy/snapRunnerNpps rename to utils/SnapPy/snapscripts/snapRunnerNpps.py index a6c59e00..175f8a9e 100755 --- a/utils/SnapPy/snapRunnerNpps +++ b/utils/SnapPy/snapscripts/snapRunnerNpps.py @@ -5,27 +5,31 @@ import multiprocessing import subprocess from Snappy.Resources import Resources +from Snappy.AddToa import add_toa_to_nc import os import sys from time import gmtime, strftime +import netCDF4 + def get_isotope_release(res, reldict): errors = "" for tag in ('releaseTime', 'radius', 'lowerHeight', 'upperHeight'): - if not tag in reldict: + if tag not in reldict: errors += "Cannot interprete {}: {}".format(tag, reldict[tag]) source_tmpl = ''' MAX.PARTICLES.PER.RELEASE= 2000 TIME.RELEASE.PROFILE.STEPS RELEASE.HOUR= 0, {releaseTime} -RELEASE.RADIUS.M= {radius}, {radius} -RELEASE.LOWER.M= {lowerHeight}, {lowerHeight} -RELEASE.UPPER.M= {upperHeight}, {upperHeight} +RELEASE.RADIUS.M= {radius} +RELEASE.LOWER.M= {lowerHeight} +RELEASE.UPPER.M= {upperHeight} ''' source_term = source_tmpl.format(releaseTime=reldict['releaseTime'], - radius=reldict['radius'], - lowerHeight=reldict['lowerHeight'], upperHeight=reldict['upperHeight']) + radius=reldict['radius'], + lowerHeight=reldict['lowerHeight'], + upperHeight=reldict['upperHeight']) isotopes = {'relI131': 'I131', 'relXE133': 'Xe133', @@ -36,7 +40,7 @@ def get_isotope_release(res, reldict): emis = float(reldict[rel]) except: pass - if (emis > 0.): + if emis > 0.0: source_term += "RELEASE.BQ/SEC.COMP= {rel}, {rel}, '{iso}'\n".format(rel=reldict[rel], iso=iso) # add Cs137, I131 and Xe133 @@ -44,6 +48,7 @@ def get_isotope_release(res, reldict): return (source_term, errors) + def runsnapsingle(npp, release_dt, tag_time, dirname, res): nPPs = res.readNPPs() reldict = { @@ -86,11 +91,9 @@ def runsnapsingle(npp, release_dt, tag_time, dirname, res): stderr=stderr ) # add time of arrival - subprocess.run(['snapAddToa', 'snap.nc'], - cwd=dirname, - stdout=stdout, - stderr=stderr - ) + fh = netCDF4.Dataset(os.path.join(dirname, "snap.nc"), mode="a") + add_toa_to_nc(fh) + # and plot prod_dir = os.path.join(dirname, 'prod') os.mkdir(prod_dir) @@ -139,22 +142,24 @@ def runsnapsingle(npp, release_dt, tag_time, dirname, res): cwd=prod_dir ) return True - -def runsnap(npps, outdir): + +def runsnap(npps, workdir, outdir): release_dt = datetime.datetime.now() + datetime.timedelta(hours=1) nppdir = {} runparams = [] res = Resources() - res.getLustreDir() # set lustredir within res before parallelization - mapp-services.sh not parallelizable yet + res.getLustreDir() # set lustredir within res before parallelization - mapp-services.sh not parallelizable yet nPPs = res.readNPPs() + if workdir is None: + workdir = res.getSnapOutputDir() for i, tag in enumerate(npps): - if not tag in nPPs: + if tag not in nPPs: print(f"unknown NPP: {tag}", file=sys.stderr) sys.exit(1) - tag_time = gmtime(release_dt.timestamp() + i) # ensure a few secs difference for tags - nppdir[tag] = os.path.join(res.getSnapOutputDir(), "{0}_{1}".format(tag, strftime("%Y-%m-%dT%H%M%S", tag_time))) + tag_time = gmtime(release_dt.timestamp() + i) # ensure a few secs difference for tags + nppdir[tag] = os.path.join(workdir, "{0}_{1}".format(tag, strftime("%Y-%m-%dT%H%M%S", tag_time))) runparams.append([tag, release_dt, tag_time, nppdir[tag], res]) with multiprocessing.Pool(4) as p: @@ -164,10 +169,10 @@ def runsnap(npps, outdir): opts = ['montage', '-mode', 'concatenate', '-tile', f"4x{len(npps)}"] for tag in npps: opts += ['-label', tag] - for t in [12,24,36,48]: + for t in [12, 24, 36, 48]: opts += [f"{nppdir[tag]}/prod/snap_{t}h.png"] opts.append(f'{outdir}/plots_{release_dt:%Y-%m-%dT%H}0000Z.png') - print('running montage with:\n'+" ".join(opts)) + print('running montage with:\n' + " ".join(opts)) subprocess.run(opts) # montage toa @@ -176,24 +181,29 @@ def runsnap(npps, outdir): opts += ['-label', tag] opts += [f"{nppdir[tag]}/prod/snap_toa.png"] opts.append(f'{outdir}/plots_{release_dt:%Y-%m-%dT%H}0000Z_toa.png') - print('running montage for toa with:\n'+" ".join(opts)) + print('running montage for toa with:\n' + " ".join(opts)) subprocess.run(opts) -if __name__ == "__main__": +def main(): os.umask(0o002) parser = argparse.ArgumentParser( description="Run snap for several Nuclear Power Plants and receive figures of dispersion", usage=f"{sys.argv[0]} NPP [NPP NPP ...]" ) + parser.add_argument("--workdir", help="output directory, default set from --store", default=None) parser.add_argument("--dir", help="output directory", default="/home/VGLSHARE/SNAP4DSA/") - parser.add_argument("--store", help="storeA or storeB, meteo and runtime-datastore, default used from MAPP-system") - parser.add_argument('NPP', help="name of Nuclear Power Plant, e.g. CHERNOBYL (see list in SnapPy)", nargs='+') + parser.add_argument("--store", help="meteo and runtime-datastore, default used from MAPP-system", choices=["storeA", "storeB"]) + parser.add_argument('NPP', help="name of Nuclear Power Plant(s), e.g. CHERNOBYL (see list in SnapPy)", nargs='+') args = parser.parse_args() - if (args.store): + if args.store: os.environ["STORE"] = args.store - runsnap(args.NPP, args.dir) + runsnap(args.NPP, args.workdir, args.dir) + + +if __name__ == "__main__": + main()