diff --git a/README.org b/README.org index 3d31d0d..87afbd5 100644 --- a/README.org +++ b/README.org @@ -3,7 +3,7 @@ #+setupfile: https://ls4gan.github.io/other/setup-page.org #+options: toc:t -The toyzero package provides first steps toward [[https://ls4gan.github.io/][LS4GAN]]. +The toyzero package provides first steps toward [[https://ls4gan.github.io/][LS4GAN]]. See also [[file:next-steps.org]] * Goals :PROPERTIES: @@ -291,6 +291,28 @@ The ~all_frames~ target generates "frame" data from depos by running the Wire-Cell simulation. The "frame" file format is described elsewhere. For here, we treat it as a temporary. +The simulation is internally structured as a DAG of components +exchanging data flow as shown: + +#+ATTR_HTML: :width 90% +[[file:plots/dots/depos-sim-adc/dag.png]] + +Frames come in different flavor depending on the final output data +tier. We name and describe them as: + +- noiseless :: these are integer ADC waveforms reflecting existence of + only ionization electrons and no electronics noise. They are + bipolar on the induction planes "U" and "V and unipolar on + collection plane "W". They have a fixed "baseline" at an ADC value + depending on the plane and are sparse (baseline-padded) at locations + away from any ionization. + +- signal :: these floating point measures of the original ionization + electrons after signal processing and noise filtering is applied to + signal and noise simulation. The waveforms from all planes are + unipolar and give a measure of the original ionization distribution. + The waveforms have a "baseline" of 0.0 and are sparse + (baseline-padded) at locations away from any ionization. ** images :PROPERTIES: @@ -308,8 +330,8 @@ match. For example: protodune-orig-0-1-W (960, 6000) #+end_example -These image arrays have shape ~(nchan, ntick)~. That is each row is the -waveform from one channel and has ~ntick=6000~ samples (3ms). When +These image arrays have shape ~(nchan, ntick)~. That is, each row is +the waveform from one channel and has ~ntick=6000~ samples (3ms). When visualized with matplotlib's ~imshow()~ you will see channels as Y-axis, tics as X-axis. @@ -319,6 +341,10 @@ This "W" file holds one array of 960 channels and 6000 sample time induction planes and each will have 800 channels and the contemporaneous 6000 ticks. +Like frames above, images come in corresponding flavors. See section +[[frames]] for description. + + * Plotting :PROPERTIES: :CUSTOM_ID: plotting @@ -547,3 +573,48 @@ made available. If the changes are generally useful, please consider making a PR! +* Production + +We run from containers. Eg, as a mongo ~docker~ command that runs +~snakemake~ in the ~toyzero~ container. + +#+begin_example + + $ docker run \ + --user user \ + --volume (pwd):/data \ + -ti ls4gan/toyzero:0.3.0 \ + "cd toyzero && \ + snakemake just_tar --notemp -j1 -p \ + --config seed=1234 outdir=/data ntracks=100 nevents=10 wcloglvl=debug threads=8" + +#+end_example + +Or, one layer down in the ~wirecell~ container, assuming you have +~toyzero~ checked out locally: + +#+begin_example + + $ cd .. # parent holding local toyzero/ + $ mkdir run + $ cd run/ + $ cp -a ../toyzero/{Snakefile,cfg,toyzero.yaml} . + $ docker run \ + --user user \ + --volume (pwd):/data \ + -ti ls4gan/wirecell:0.16.0 \ + "cd /data && \ + snakemake just_tar -j1 -p \ + --config seed=1234 ntracks=100 nevents=10 wcloglvl=debug threads=8" + $ ls -l toyzero-100-10-1234.tar + +#+end_example + +Note: will likely want to set ~threads=1~ for batch and *MUST* set +~seed=XYZ~ uniquely for each submission. The ~outdir~ setting may be used +if output should not go to the CWD. The tar file base name can be +controlled with ~outname~. Intermediate files that eventually go in to +the tar file land in ~{outdir}/seed-{seed}/~. The ~./.snakemake/~ +directory receives snakemake control and log files. If run with +~--notemp~ then (ironically, should be named ~--yestemp~) temporary files +marked with ~temp()~ in the Snakefile are *kept*. diff --git a/Snakefile b/Snakefile index 3f87317..790e99a 100644 --- a/Snakefile +++ b/Snakefile @@ -1,7 +1,7 @@ #!/usr/bin/env snakemake -import json import os +import json from snakemake.remote.HTTP import RemoteProvider as HTTPRemoteProvider HTTP = HTTPRemoteProvider() @@ -13,18 +13,38 @@ configfile: "toyzero.yaml" ## $ snakemake --config ntracks=100 [...] datadir = config.get("datadir", "data") plotdir = config.get("plotdir", "plots") +logdir = config.get("logdir", "logs") outdir = config.get("outdir", os.environ.get("TOYZERO_OUTDIR", ".")) -datadir = os.path.abspath(os.path.join(outdir, datadir)) -plotdir = os.path.abspath(os.path.join(outdir, plotdir)) seed = config.get("seed", "1,2,3,4") -ntracks = config.get("ntracks", 10) -nevents = config.get("nevents", 10) +ntracks = int(config.get("ntracks", 10)) +nevents = int(config.get("nevents", 10)) +depotimes = config.get("depotimes", "-3*ms,6*ms") wcloglvl = config.get("wcloglvl", "info") -# print(f"OUTDIR:{outdir}") -# print(f"NEVENTS:{nevents}") -# print(f"WCLOGLVL:{wcloglvl}") + +seedlst_ = str(seed).replace(",","-") +seeddir = f'seed-{seedlst_}' +outuniq = os.path.abspath(os.path.join(outdir, seeddir)) +datadir = os.path.abspath(os.path.join(outuniq, datadir)) +plotdir = os.path.abspath(os.path.join(outuniq, plotdir)) +logdir = os.path.abspath(os.path.join(outuniq, logdir)) + +outname = config.get("outname", f'{outdir}/toyzero-{ntracks}-{nevents}-{seed}') + +# Which intermediate data tier(s) to save from the full simulation +# chain. It may include "orig". The "gauss" tier will always be included. +frame_tap_tiers = list(config.get("frame_tap_tiers", ('orig', ))) +if "gauss" not in frame_tap_tiers: + frame_tap_tiers += ["gauss"] + + +# limit number of threads given to a wire-cell job +wct_threads = int(config.get("threads", 1)) + +# single threaded uses Pgrapher, multi uses TbbFlow +wct_threading = "single" if wct_threads == 1 else "multi" +# print(f'WCT THREADS {wct_threads} ({wct_threading})') # The rest are hard wired for now wcdata_url = "https://github.com/WireCell/wire-cell-data/raw/master" @@ -38,6 +58,18 @@ wcdata_ext = "json.bz2" ## to span some set of each. resps = "dune-garfield-1d565" wires = "protodune-wires-larsoft-v4" +# a few places want a list of APA IDs +apa_iota = list(range(6)) + +# The data domains describe a universe. For toyzero, they are +# associated with a particular detector response. +DOMAINS = ["fake", "real"] + +# The data tiers for frame or image. We make all tiers in one job. +# The 'orig' is the output of the simulation and 'gauss' is the output +# of signal processing using the charge-preserving filters. +SIM_TIERS = ["orig", "gauss"] +TIERS = SIM_TIERS + ["splat"] # some important file names real_resps = f'{datadir}/resps/real-resps.{wcdata_ext}' @@ -45,7 +77,7 @@ fake_resps = f'{datadir}/resps/fake-resps.{wcdata_ext}' domain_resps = f'{datadir}/resps/{{domain}}-resps.{wcdata_ext}' wires_file = f'{datadir}/wires/wires.{wcdata_ext}' depos_file = f'{datadir}/depos/depos.npz' -domain_frames = f'{datadir}/frames/{{domain}}-frames.npz' +noise_file = f'{datadir}/noise/protodune-noise-spectra-v1.{wcdata_ext}' # resp - prepare response files @@ -67,6 +99,15 @@ rule gen_resp_fake: wirecell-sigproc frzero -n 0 -o {output} {input} ''' +rule get_noisef: + input: + HTTP.remote(f'{wcdata_url}/protodune-noise-spectra-v1.{wcdata_ext}', keep_local=True) + output: + temp(noise_file) + run: + shell("mkdir -p {datadir}") + shell("cp {input} {output}") + rule plot_resp: input: domain_resps @@ -77,10 +118,14 @@ rule plot_resp: wirecell-sigproc plot-response {input} {output} ''' -rule all_resp: +rule just_resp: input: rules.get_resp_real.output, rules.gen_resp_fake.output, + +rule all_resp: + input: + rules.just_resp.input, expand(rules.plot_resp.output, domain=["real","fake"]) # wires - get wires file @@ -105,48 +150,32 @@ rule plot_wires: wirecell-util plot-wires {input} {output} ''' -rule all_wires: +rule just_wires: input: rules.get_wires.output, + +rule all_wires: + input: + rules.just_wires.input, rules.plot_wires.output # depos - generate ionization point depositions -def gen_depos_cfg(w): - 'Dig out the bounding box of the detector' - - params_cmd = 'wcsonnet pgrapher/experiment/pdsp/simparams.jsonnet' - jtext = subprocess.check_output(params_cmd, shell=True) - jdat = json.loads(jtext) - bb = jdat['det']['bounds'] - - p1 = bb['tail'] - p2 = bb['head'] - corn = list() - diag = list() - for l in "xyz": - c = p1[l] - dc = p2[l]-p1[l] - # warning, pretend we know WCT's SoU here.... - corn.append(f'{c:.1f}*mm') - diag.append(f'{dc:.1f}*mm') - - return dict(tracks = ntracks, sets = nevents, seed = seed, - corn = ','.join(corn), diag = ','.join(diag)) - rule gen_depos: input: wires_file params: - p = gen_depos_cfg + diag = '16000.0*mm,6100.0*mm,7000.0*mm', + corn = '-8000.0*mm,0.0*mm,0.0*mm' output: temp(depos_file) shell: ''' wirecell-gen depo-lines \ - --seed {params.p[seed]} \ - --tracks {params.p[tracks]} --sets {params.p[sets]} \ - --diagonal '{params.p[diag]}' --corner '{params.p[corn]}' \ + --time {depotimes} \ + --seed {seed} \ + --tracks {ntracks} --sets {nevents} \ + --diagonal '{params.diag}' --corner '{params.corn}' \ --output {output} ''' @@ -159,88 +188,205 @@ rule plot_depos: wirecell-gen plot-sim {input} {output} -p depo ''' +rule just_depos: + input: + expand(rules.gen_depos.output, wire=wires) + + rule all_depos: input: - expand(rules.gen_depos.output, wire=wires), + rules.just_depos.input, expand(rules.plot_depos.output, wire=wires) # frames -rule sim_dots: + +wct_sigproc_cfg = 'cfg/main-depos-sigproc.jsonnet' +wct_splat_cfg = 'cfg/main-depos-splat.jsonnet' + +def wct_dots_params(w): + if w.verb == "brief": + return "--no-params" + return "" +# note, we pass bogus TLAs so don't use the generated json for +# anything real! +rule wct_dots: input: - config = 'cfg/main-depos-sim-adc.jsonnet' + config = wct_sigproc_cfg output: - json = f'{plotdir}/sim-graph.json', - dot = f'{plotdir}/sim-graph.dot', - png = f'{plotdir}/sim-graph.png', - pdf = f'{plotdir}/sim-graph.pdf' + json = temp(f'{plotdir}/dots/cfg-{{verb}}.json'), + dot = temp(f'{plotdir}/dots/dag-{{verb}}.dot'), + png = f'{plotdir}/dots/dag-{{verb}}.png', + pdf = f'{plotdir}/dots/dag-{{verb}}.pdf' + params: + wct_dots_params shell: ''' + mkdir -p {plotdir}/dots; wcsonnet \ -P cfg \ -A input=DEPOS-FILE \ - -A output=FRAMES-FILE \ + --tla-code taps='{{"orig":"frame-orig-apa%d.tar.bz2","gauss":"frame-gauss-apa%d.tar.bz2"}}' \ -A wires=WIRES-FILE \ - -A resps=RESPS-FILE \ + -A resps_sim=RESPS-SIM-FILE \ + -A resps_sigproc=RESPS-SIGPROC-FILE \ + -A noisef=NOISE-FILE \ {input.config} > {output.json}; - wirecell-pgraph dotify --jpath=-1 {output.json} {output.dot} ; + wirecell-pgraph dotify {params} --jpath=-1 {output.json} {output.dot} ; dot -Tpng -o {output.png} {output.dot} ; dot -Tpdf -o {output.pdf} {output.dot} ''' +rule all_dots: + input: + expand(rules.wct_dots.output, verb=["full", "brief"]) + +# this gives the pattern for one per-APA frame file. The %d is +# interpolated by wire-cell configuration. + +frames_ext = "tar.bz2" +frames_wildcard = f'{datadir}/frames/{{tier}}/{{sim_domain}}-{{sigproc_domain}}-frames-apa{{apa}}.{frames_ext}' + +def frame_taps(w): + frames_pattern = f'{datadir}/frames/{{tier}}/{{sim_domain}}-{{sigproc_domain}}-frames-apa%d.{frames_ext}' + taps = list() + for tier in frame_tap_tiers: + d = dict(w) + d["tier"] = tier + taps.append('"%s":"%s"' % (tier, frames_pattern.format(**d))) + + taps = ",".join(taps) + return '{%s}'%taps + +def sim_frame_files(): + frames_pattern = datadir + '/frames/{tier}/{{sim_domain}}-{{sigproc_domain}}-frames-apa{apaid}.' + frames_ext + return expand(frames_pattern, tier=SIM_TIERS, apaid = apa_iota) rule sim_frames: input: wires = wires_file, - resps = domain_resps, + resps_sim = f'{datadir}/resps/{{sim_domain}}-resps.{wcdata_ext}', + resps_sigproc = f'{datadir}/resps/{{sigproc_domain}}-resps.{wcdata_ext}', depos = depos_file, - config = 'cfg/main-depos-sim-adc.jsonnet' + config = wct_sigproc_cfg, + noise = noise_file output: - frames = temp(domain_frames) + temp(sim_frame_files()) + params: + taps = frame_taps + log: + f'{logdir}/sim-frames-{{sim_domain}}-{{sigproc_domain}}.log' + benchmark: + f'{logdir}/sim-frames-{{sim_domain}}-{{sigproc_domain}}.tsv' shell: ''' - rm -f {output.frames}; + rm -f {output}; wire-cell \ - -l stdout -L {config[wcloglvl]} \ + --threads {wct_threads} \ + -A thread={wct_threading} \ + -l {log} -L {config[wcloglvl]} \ -P cfg \ -A input={input.depos} \ - -A output={output.frames} \ + --tla-code 'taps={params.taps}' \ -A wires={input.wires} \ - -A resps={input.resps} \ + -A resps_sim={input.resps_sim} \ + -A resps_sigproc={input.resps_sigproc} \ + -A noisef={input.noise} \ -c {input.config} ''' + +# like sim_frames but we use DepoSplat instead of sim+sigproc. +rule splat_frames: + input: + wires = wires_file, + depos = depos_file, + config = wct_splat_cfg + output: + temp([f'{datadir}/frames/splat/splat-frames-apa{apa}.{frames_ext}' for apa in apa_iota]) + params: + taps = f'{{"splat":"{datadir}/frames/splat/splat-frames-apa%d.{frames_ext}"}}' + log: + f'{logdir}/splat-frames.log' + benchmark: + f'{logdir}/splat-frames.tsv' + shell: ''' + rm -f {output}; + wire-cell \ + --threads {wct_threads} \ + -A thread={wct_threading} \ + -l {log} -L {config[wcloglvl]} \ + -P cfg \ + -A input={input.depos} \ + --tla-code 'taps={params.taps}' \ + -A wires={input.wires} \ + -c {input.config} + ''' + + def gen_plot_frames(w): i = int(w.apa) - return dict(chb=f'{i},{i+2560}') + tag = f"{w.tier}{i}" + return dict(chb=f'0,2560', tag=tag) rule plot_frames: input: - domain_frames + frames_wildcard output: - f'{plotdir}/frames-{{domain}}-apa{{apa}}.{{ext}}' + f'{plotdir}/frames/{{tier}}/{{sim_domain}}-{{sigproc_domain}}-apa{{apa}}.{{ext}}' params: p = gen_plot_frames shell:''' - wirecell-gen plot-sim {input} {output} -p frames -b {params.p[chb]} + wirecell-gen plot-sim {input} {output} -p frames -b {params.p[chb]} --tag "{params.p[tag]}" ''' -rule plot_frames_hidpi: + +rule plot_splat_frames: input: - f'{plotdir}/frames-{{domain}}-apa{{apa}}.pdf' + f'{datadir}/frames/splat/splat-frames-apa{{apa}}.tar.gz' output: - f'{plotdir}/hidpi/frames-{{domain}}-apa{{apa}}.png' - params: - + f'{plotdir}/frames/splat/splat-apa{{apa}}.{{ext}}' shell:''' - pdftoppm -rx 600 -ry 600 {input} | pnmtopng > {output} + wirecell-gen plot-sim {input} {output} -p frames -b 0,2560 --tag splat{wildcards.apa} ''' +rule just_splat_frames: + input: + rules.splat_frames.output + +rule all_splat_frames: + input: + rules.just_splat_frames.input, + expand(rules.plot_splat_frames.output, + ext=["png"], apa=apa_iota) + + +rule just_frames: + input: + expand(rules.sim_frames.output, + sim_domain=["real"], + sigproc_domain=["fake"], + ), + expand(rules.sim_frames.output, + sim_domain=["fake"], + sigproc_domain=["fake"], + ) + + rule all_frames: input: - expand(rules.sim_frames.output, domain=["real","fake"]), - expand(rules.plot_frames.output, domain=["real","fake"], - ext=["png","pdf"], apa=list(range(6))), - expand(rules.plot_frames_hidpi.output, domain=["real","fake"], - apa=[0]) + rules.all_resp.input, + rules.all_wires.input, + rules.all_depos.input, + rules.just_frames.input, + expand(rules.plot_frames.output, + sim_domain=["real","fake"], + sigproc_domain=["real","fake"], + tier=SIM_TIERS, + ext=["png"], apa=apa_iota), + + # expand(rules.plot_frames_hidpi.output, + # sim_domain=["real","fake"], + # sigproc_domain=["real","fake"], + # tier=TIERS, apa=[0]) @@ -255,65 +401,211 @@ rule all_frames: ## static data (ie, defined right here). So, that is what we do. # split_outer_product = dict( - domain = ["protodune"], event = list(range(nevents)), - apa = list(range(6)), plane = ["U","V","W"], ) +# frames_wildcard = f'{datadir}/frames/{{tier}}/{{domain}}-frames-apa{{apa}}.{frames_ext}' + +# this pattern is formatted inside wirecell-util frame-split. +split_array_pattern = '{detector}-{tag}-{index}-{planeletter}' + +sigproc_rebin = 4 +def rebin_number(w): + if w.tier == "orig": + return 0; + return sigproc_rebin +def tick_offset(w): + if w.tier == 'splat': + # approx, chosen by comparing gauss W to splat W plots + # this is pre-rebin ticks, 0.5us. + return 125+8 + return 0 + rule split_images: input: - domain_frames + frames_wildcard output: - expand(datadir+'/images/{{domain}}/protodune-orig-{event}-{apa}-{plane}.npz', + expand(datadir+'/images/{{tier}}/{{sim_domain}}-{{sigproc_domain}}/protodune-{{tier}}{{apa}}-{event}-{plane}.npz', **split_outer_product) - shell: ''' - wirecell-util frame-split \ - -f {datadir}/images/{wildcards.domain}/{{detector}}-{{tag}}-{{index}}-{{anodeid}}-{{planeletter}}.npz \ - {input} - ''' - -def gen_title(w): - if w.domain == 'real': - dim='2D' + params: + outpath = datadir+'/images/{tier}/{sim_domain}-{sigproc_domain}', + mdpath = 'metadata-{apa}.json', + rebin = rebin_number, + tickoff = tick_offset + benchmark: + f'{logdir}/split-images-{{tier}}-{{sim_domain}}-{{sigproc_domain}}-apa{{apa}}.tsv' + run: + if not os.path.exists(params.outpath): + os.makedirs(params.outpath) + mdpath = os.path.join(params.outpath, params.mdpath) + with open(mdpath, "w") as fp: + fp.write(json.dumps(dict(wildcards), indent=4)) + shell('wirecell-util frame-split -t {params.tickoff} -r {params.rebin} -m {mdpath} -a {params.outpath}/{split_array_pattern} {input}') + +rule split_splat_images: + input: + f'{datadir}/frames/splat/splat-frames-apa{{apa}}.{frames_ext}' + output: + expand(datadir+'/images/{{tier}}/splat/protodune-{{tier}}{{apa}}-{event}-{plane}.npz', + **split_outer_product) + params: + outpath = datadir+'/images/{tier}/splat', + mdpath = 'metadata-{apa}.json', + rebin = rebin_number, + tickoff = tick_offset + run: + if not os.path.exists(params.outpath): + os.makedirs(params.outpath) + mdpath = os.path.join(params.outpath, params.mdpath) + with open(mdpath, "w") as fp: + fp.write(json.dumps(dict(wildcards), indent=4)) + shell('wirecell-util frame-split -t {params.tickoff} -r {params.rebin} -m {mdpath} -a {params.outpath}/{split_array_pattern} {input}') + + +def plot_split_params(w): + if w.tier == 'splat': + dim="splat" + elif w.sim_domain == 'real': + dim='sim:2D' + else: + dim='sim:q1D' + + if w.tier == "gauss": + # only one response is relevant to ADC tier + # signals may have a different response used in decon. + if w.sigproc_domain == 'real': + dim+='/SP:2D' + else: + dim+='/SP:q1D' + + if w.tier == "orig": + ztitle='ADC (baseline subtracted)' + vmin=-50 + vmax=50 + baseline="median" else: - dim='q1D' - return f'"{w.domain}/{dim}, event {w.event}, APA {w.apa}, {w.plane} plane"', + ztitle='Signal (ionization electrons)' + vmin=0 + vmax=20000 + baseline="0" + + # we make 2 zoom levels. 1 is full, 2 is something zoomed in. + + chan0 = 0 + nchans = 960 if w.plane == "W" else 800 + tick0 = 0 + nticks = 6000 + + # pick a region with good activity in all 3 views + if int(w.zoomlevel) == 2: + nchans = int(nchans / 5) + tick0 = 3600 + nticks = 800 + + if w.tier != "orig": + tick0 = int(tick0/sigproc_rebin) + nticks = int(nticks/sigproc_rebin) + + zoom = f'{chan0}:{chan0+nchans},{tick0}:{tick0+nticks}' + title = f'{w.tier} {dim}, event {w.event}, APA {w.apa}, {w.plane} plane' + + return locals() + + ## Note, we must match the input here by hand to the output above ## because the domain is not included in the expand above but is here. ## Above is 1->N, here is 1->1. -rule plot_split_images: - input: - datadir+'/images/{domain}/protodune-orig-{event}-{apa}-{plane}.npz', - output: - plotdir+'/images/{domain}/{cmap}/protodune-orig-{event}-{apa}-{plane}.{ext}' - params: - title = gen_title - shell: ''' +plot_split_shell = ''' wirecell-util npz-to-img --cmap {wildcards.cmap} \ - --title {params.title} \ + --title '{params.p[title]}' \ --xtitle 'Relative ticks number' \ --ytitle 'Relative channel number' \ - --ztitle 'ADC (baseline subtracted)' \ - --zoom 300:500,0:1000 --mask 0 --vmin -50 --vmax 50 \ - --dpi 600 --baseline=median -o {output} {input} + --ztitle '{params.p[ztitle]}' \ + --zoom '{params.p[zoom]}' \ + --vmin '{params.p[vmin]}' --vmax '{params.p[vmax]}' \ + --mask 0 \ + --dpi 600 --baseline='{params.p[baseline]}' -o {output} {input} ''' +rule plot_split_images: + input: + datadir+'/images/{tier}/{sim_domain}-{sigproc_domain}/protodune-{tier}{apa}-{event}-{plane}.npz', + output: + plotdir+'/images/{tier}/{sim_domain}-{sigproc_domain}/{cmap}/protodune-{tier}{apa}-{event}-{plane}-zoom{zoomlevel}.{ext}' + params: + p = plot_split_params + shell: plot_split_shell + +rule plot_split_splat_images: + input: + datadir+'/images/{tier}/splat/protodune-{tier}{apa}-{event}-{plane}.npz', + output: + plotdir+'/images/{tier}/splat/{cmap}/protodune-{tier}{apa}-{event}-{plane}-zoom{zoomlevel}.{ext}' + params: + p = plot_split_params + shell: plot_split_shell + +rule just_images: + input: + rules.just_frames.input, + expand(rules.split_images.output, + sim_domain=["real"], + sigproc_domain=["fake"], + tier=["gauss"], apa=apa_iota), + expand(rules.split_images.output, + sim_domain=["fake"], + sigproc_domain=["fake"], + tier=["gauss"], apa=apa_iota) + +rule just_splat_images: + input: + rules.just_splat_frames.input, + expand(rules.split_splat_images.output, + tier=["splat"], apa=apa_iota) + ## note, list-of-list for the split_images rule rule all_images: input: - expand(rules.split_images.output, domain=["real","fake"]), + rules.just_images.input, + expand( + rules.plot_split_images.output, + sim_domain = ["fake"], + sigproc_domain = ["fake"], + tier = ["gauss"], + event = [0], apa=apa_iota, plane=["U","V","W"], + ext = ["png"], + cmap = ["viridis"], + zoomlevel=[1, 2], + ), expand( rules.plot_split_images.output, - domain = ["real","fake"], - event = [0], apa=[2], plane=["U"], - ext = ["png", "pdf", "svg"], - cmap = ["seismic", "Spectral", "terrain", "coolwarm", "viridis"], + sim_domain = ["real"], + sigproc_domain = ["fake"], + tier = ["gauss"], + event = [0], apa=apa_iota, plane=["U","V","W"], + ext = ["png"], + cmap = ["viridis"], + zoomlevel=[1, 2], ) -rule just_images: +rule all_splat: input: - expand(rules.split_images.output, domain=["real","fake"]), + rules.just_splat_images.input, + expand( + rules.plot_split_splat_images.output, + tier = ["splat"], + event = [0], apa=apa_iota, plane=["U","V","W"], + ext = ["png"], + cmap = ["viridis"], + zoomlevel=[1, 2], + ) + + +rule just: + input: + rules.just_frames.input, + rules.just_images.input rule all: input: @@ -323,3 +615,17 @@ rule all: rules.all_frames.input, rules.all_images.input +rule just_tar: + input: + rules.just.input, + output: + f'{outname}.tar' + shell: ''' + tar --exclude data/depos \ + --exclude data/noise \ + --exclude data/wires \ + --exclude data/resps \ + --exclude data/frames/orig \ + -cf {output} {outuniq} + ''' + diff --git a/cfg/ioutils.jsonnet b/cfg/ioutils.jsonnet index 0688771..43abb84 100644 --- a/cfg/ioutils.jsonnet +++ b/cfg/ioutils.jsonnet @@ -1,8 +1,11 @@ local wc = import "wirecell.jsonnet"; -local g = import 'pgraph.jsonnet'; +local pg = import 'pgraph.jsonnet'; + +local ut = import 'utils.jsonnet'; + { // Return a numpy depo source configuration. - npz_source_depo(name, filename) :: g.pnode({ + npz_source_depo(name, filename) :: pg.pnode({ type: 'NumpyDepoLoader', name: name, data: { @@ -12,7 +15,7 @@ local g = import 'pgraph.jsonnet'; // Return a numpy frame saver configuration. - frame_save_npz(name, filename, digitize=true, tags=[]) :: g.pnode({ + frame_save_npz(name, filename, digitize=true, tags=[]) :: pg.pnode({ type: 'NumpyFrameSaver', name: name, data: { @@ -22,8 +25,62 @@ local g = import 'pgraph.jsonnet'; }}, nin=1, nout=1), // Use to cap off a frame stream with a sink. - frame_sink(name="frame-sink") :: g.pnode({ + frame_cap(name="frame-cap") :: pg.pnode({ type: "DumpFrames", name: name, }, nin=1, nout=0), + + + // Sink a frame to a tar stream. Filename extension can be .tar, + // .tar.bz2 or .tar.gz. There's no monkey business with a %d in + // the file name. Pass in a unique, literal file name. Same goes + // for tags. + frame_sink(name, outfile, tags=[], digitize=false) :: + pg.pnode({ + type: "FrameFileSink", + name: name, + data: { + outname: outfile, + tags: tags, + digitize: digitize, + }, + }, nin=1, nout=0), + + frame_tap(name, sink, tag, cap=false) :: + if cap + then sink + else pg.fan.tap('FrameFanout', sink, name, + tag_rules=[ // one for each port! + {frame:{'.*':tag}}, + {frame:{'.*':tag}}, + ]), + + // Save frames to outfile. + // + // name, outfile and elements of the tags list may have a single + // "%" format code which if it exists will be interpolated again + // the "index". + // + // If outfile has not "%" format the string "-apa%d" will be + // appended to the base file name (prior to .ext). + // + // The "name" is used to give unique objects. + // + // The tags determine which among all available frames/trace tags + // to save. + // + // If digitize is true, frame samples will be truncated to int + // else left as float. + // + // If cap is false, the resulting node acts as a filter. + frame_out(name, index, outfile, tags=["gauss%d"], digitize=false, cap=true) :: { + local nam = if ut.haspct(name) then name%index else name, + local tint = [if ut.haspct(t) then t%index else t for t in tags], + local end = if cap then [$.frame_cap(nam)] else [], + local outf = if ut.haspct(outfile) then outfile%index else ut.basename_append(outfile, "-apa%d"%index), + + ret: pg.pipeline([$.frame_save_npz(nam, outf, digitize=digitize, tags=tint)] + + end) + }.ret, + } diff --git a/cfg/main-depos-sigproc.jsonnet b/cfg/main-depos-sigproc.jsonnet new file mode 100644 index 0000000..1241cac --- /dev/null +++ b/cfg/main-depos-sigproc.jsonnet @@ -0,0 +1,111 @@ +// This is a main wire-cell configuration file. +// +// It implements the chain: +// (depos)->sim(signal,noise)->sigproc->(signals) +// +// It takes a number of top-level-arguments (TLA) which can control +// input configuration and input data and a number of output data +// "taps". You may also control if it runs in single or multi thread +// mode +// +// Run like +// +// wire-cell \ +// -A depos=depos.npz \ +// -A taps={...see below...} \ +// -A wires=wires-geometry.json.bz2 \ +// -A resps_sim=field-response-for-sim.json.bz2 \ +// -A resps_sigproc=field-response-for-sigproc.json.bz2 \ +// -A noisef=/path/to/noise/model/file \ +// -A thread=[single|multi] \ +// -c main-depos-sigproc.jsonnet +// +// The 'noisef' TLA is optional and specifies a noise model file. If +// not given, only signal is simulated. +// +// If "taps" is given it specifies a mapping from a data tier key word +// to a file pattern. The key is one of the conventional tags: +// +// - orig :: means the ADC-level frames out of the simulation +// - gauss :: means the signal processing with Gaussian filter +// +// The file pattern MUST have a %d format marker which will be +// interpolated on the APA ID. + + +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; + +local tz = import "toyzero.jsonnet"; +local io = import "ioutils.jsonnet"; +local nf = import "nf.jsonnet"; +local sp = import "sp.jsonnet"; + +local params = import "pgrapher/experiment/pdsp/simparams.jsonnet"; +local spfilt = import "pgrapher/experiment/pdsp/sp-filters.jsonnet"; +local chndb = import "pdsp_chndb.jsonnet"; + +function(input, taps, wires, resps_sim, resps_sigproc, noisef=null, thread='single') + local app = if thread == 'single' + then 'Pgrapher' + else 'TbbFlow'; + local seeds = [0,1,2,3,4]; // maybe let CLI set? + local depos = io.npz_source_depo("depos", input); + + + local wireobj = tz.wire_file(wires); + local anodes = tz.anodes(wireobj, params.det.volumes); + + local apaids = std.range(0, std.length(anodes)-1); + + local robjs_sim = tz.responses(resps_sim, params.elec, params.daq); + local robjs_sigproc = tz.responses(resps_sigproc, params.elec, params.daq); + + local random = tz.random(seeds); + + local drifter = tz.drifter(params.det.volumes, params.lar, random); + local bagger = tz.bagger(params.daq); + + local chndb_perfect(n) = + chndb.perfect(anodes[n], robjs_sim.fr, + params.daq.nticks, + params.daq.tick); + + local tap_out(tap, apaid, cap=false) = { + local name = "%s%d"%[tap,apaid], + local digi = tap == "raw" || tap == "orig", + res: if std.objectHas(taps, tap) + then [io.frame_tap(name, io.frame_sink(name, taps[tap]%apaid, tags=[name], digitize=digi), name, cap)] + else [] + }.res; + + local sim(n) = [ + local anode = anodes[n]; + tz.sim(anode, // kitchen + robjs_sim.pirs, // sink + params.daq, + params.adc, + params.lar, + noisef, + 'adc', + random) + ] + tap_out("orig", n); + + local adcpermv = tz.adcpermv(params.adc); + local nfsp(n) = [ + nf(anodes[n], robjs_sigproc.fr, chndb_perfect(n), + params.daq.nticks, params.daq.tick), + ] + tap_out("raw", n) + [ + sp(anodes[n], robjs_sigproc.fr, robjs_sigproc.er, spfilt, adcpermv) + ] + tap_out("gauss", n, true); + + local oneapa(n) = pg.pipeline(sim(n) + nfsp(n)); + + local pipes = [oneapa(n) for n in apaids]; + + local body = pg.fan.fanout('DepoSetFanout', pipes); + local full = pg.pipeline([depos, drifter, bagger, body]); + + tz.main(full, app) + + diff --git a/cfg/main-depos-sim-adc.jsonnet b/cfg/main-depos-sim-adc.jsonnet index 6aceacb..63b09f6 100644 --- a/cfg/main-depos-sim-adc.jsonnet +++ b/cfg/main-depos-sim-adc.jsonnet @@ -10,7 +10,7 @@ // -A input=depos.npz -A output=frames.npz \ // -A wires=wires-geometry.json.bz2 \ // -A resps=field-response.json.bz2 \ -// -A noise="yes" \ +// -A noisef=/path/to/noise/model/file \ // -c main-depos-sim-adc.jsonnet // // The 'noise' TLA is optional and defaults to "no". @@ -21,36 +21,35 @@ local pg = import "pgraph.jsonnet"; local tz = import "toyzero.jsonnet"; local io = import "ioutils.jsonnet"; -function(input, output, wires, resps, noisef=null) +function(input, output, wires, resps, noisef=null, app='Pgrapher') local seeds = [0,1,2,3,4]; // maybe let CLI set? local depos = io.npz_source_depo("depos", input); -local params = tz.protodune_params; +local params = import "pgrapher/experiment/pdsp/simparams.jsonnet"; local wireobj = tz.wire_file(wires); local anodes = tz.anodes(wireobj, params.det.volumes); local apaids = std.range(0, std.length(anodes)-1); -local pirs = tz.pirs(resps, params.daq, params.elec); +local robjs = tz.responses(resps, params.elec, params.daq); local random = tz.random(seeds); local drifter = tz.drifter(params.det.volumes, params.lar, random); local bagger = tz.bagger(params.daq); -local apasim(n) = - tz.apasim(anodes[n], pirs, params.det.volumes[n], - params.daq, params.adc, params.lar, - noisef, random); - -local simpipes = [apasim(n) for n in apaids]; +local oneapa(n) = + tz.sim(anodes[n], robjs.pirs, + params.daq, params.adc, params.lar, + noisef, 'adc', random); +local pipes = [oneapa(n) for n in apaids]; local frames = io.frame_save_npz("frames", output); local dump = io.frame_sink(); -local simbody = pg.fan.pipe('DepoSetFanout', simpipes, 'FrameFanin'); +local simbody = pg.fan.pipe('DepoSetFanout', pipes, 'FrameFanin'); local full = pg.pipeline([depos, drifter, bagger, simbody, frames, dump]); -tz.main(full) +tz.main(full, app) diff --git a/cfg/main-depos-sim.jsonnet b/cfg/main-depos-sim.jsonnet new file mode 100644 index 0000000..e3a9df4 --- /dev/null +++ b/cfg/main-depos-sim.jsonnet @@ -0,0 +1,63 @@ +// This is a main wire-cell configuration file. +// +// It implements the chain (depos)->[sim[+noise]]->(Voltage|ADC). +// +// It takes a number of top-level-arguments (TLA). +// +// Run like +// +// wire-cell \ +// -A input=depos.npz -A output=frames.npz \ +// -A wires=wires-geometry.json.bz2 \ +// -A resps=field-response.json.bz2 \ +// -A noisef=/path/to/noise/model/file \ +// -A tier=[adc|voltage] \ +// -A app=[Pgrapher|TbbFlow] \ +// -c main-depos-sim-adc.jsonnet +// +// The 'noisef' TLA is optional and specifies a noise model. If not +// given, only signal is simulated. +// +// The 'tier' is optional and default to 'adc'. If 'voltage' is +// given, the output frame will not have the ADC model applied and +// will be in units of Volts instead of ADC counts. + +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; + +local tz = import "toyzero.jsonnet"; +local io = import "ioutils.jsonnet"; + +function(input, output, wires, resps, noisef=null, tier='adc', app='Pgrapher') +local seeds = [0,1,2,3,4]; // maybe let CLI set? +local depos = io.npz_source_depo("depos", input); + +local params = tz.protodune_params; + +local wireobj = tz.wire_file(wires); +local anodes = tz.anodes(wireobj, params.det.volumes); + +local apaids = std.range(0, std.length(anodes)-1); + +local pirs = tz.pirs(resps, params.daq, params.elec); + +local random = tz.random(seeds); + +local drifter = tz.drifter(params.det.volumes, params.lar, random); +local bagger = tz.bagger(params.daq); + +local apasim(n) = + tz.apasim(anodes[n], pirs, + params.daq, params.adc, params.lar, + noisef, tier, random); + +local simpipes = [apasim(n) for n in apaids]; + + +local frames = io.frame_save_npz("frames", output, digitize = tier=='adc'); +local dump = io.frame_sink(); + +local simbody = pg.fan.pipe('DepoSetFanout', simpipes, 'FrameFanin'); +local full = pg.pipeline([depos, drifter, bagger, simbody, frames, dump]); +tz.main(full, app) + diff --git a/cfg/main-depos-splat.jsonnet b/cfg/main-depos-splat.jsonnet new file mode 100644 index 0000000..9264ac3 --- /dev/null +++ b/cfg/main-depos-splat.jsonnet @@ -0,0 +1,78 @@ +// This is a main wire-cell configuration file. +// +// It implements the chain: +// (depos)->splat->(signals) +// +// It takes a number of top-level-arguments (TLA) which can control +// input configuration and input data and a number of output data +// "taps". You may also control if it runs in single or multi thread +// mode +// +// Run like +// +// wire-cell \ +// -A depos=depos.npz \ +// -A taps={...see below...} \ +// -A wires=wires-geometry.json.bz2 \ +// -A thread=[single|multi] \ +// -c main-depos-splat.jsonnet +// +// If "taps" is given it specifies a mapping from a data tier key word +// to a file name. The key is one of the conventional tags: +// +// - splat :: the output of DepoSplat. +// +// The file name must have a "%d" formatter which will be interpolated +// against the APA ID number. Tap files likely one of .tar, .tar.gz +// or .tar.bz2. + +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; + +local tz = import "toyzero.jsonnet"; +local io = import "ioutils.jsonnet"; + +local params = import "pgrapher/experiment/pdsp/simparams.jsonnet"; +local spfilt = import "pgrapher/experiment/pdsp/sp-filters.jsonnet"; +local chndb = import "pdsp_chndb.jsonnet"; + +function(input, taps, wires, thread='single') + local app = if thread == 'single' + then 'Pgrapher' + else 'TbbFlow'; + local seeds = [0,1,2,3,4]; // maybe let CLI set? + local depos = io.npz_source_depo("depos", input); + + + local wireobj = tz.wire_file(wires); + local anodes = tz.anodes(wireobj, params.det.volumes); + + local apaids = std.range(0, std.length(anodes)-1); + + local random = tz.random(seeds); + + local drifter = tz.drifter(params.det.volumes, params.lar, random); + + local tap_out(tap, apaid, cap=true) = { + local name = "%s%d"%[tap,apaid], + local digi = tap == "raw" || tap == "orig", + res: if std.objectHas(taps, tap) + then [io.frame_tap(name, io.frame_sink(name, taps[tap]%apaid, tags=[name], digitize=digi), name, cap)] + else [] + }.res; + + local splat(n) = [ + local anode = anodes[n]; + tz.splat(anode, params.daq, params.lar, random) + ] + tap_out("splat", n); + + local oneapa(n) = pg.pipeline(splat(n)); + + local pipes = [oneapa(n) for n in apaids]; + + local body = pg.fan.fanout('DepoFanout', pipes); + local full = pg.pipeline([depos, drifter, body]); + + tz.main(full, app) + + diff --git a/cfg/nf.jsonnet b/cfg/nf.jsonnet new file mode 100644 index 0000000..ce107b8 --- /dev/null +++ b/cfg/nf.jsonnet @@ -0,0 +1,55 @@ +// Produce a noise filter component for an anode + +// This can be arbitrarily hairy and probably best to make a new file +// if substantial changes are needed. + +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; + +function(anode, fieldresp, chndb, nsamples, tick=0.5*wc.us, rms_cuts=[]) { + local apaid = anode.data.ident, + + local single = { + type: 'pdOneChannelNoise', + name: 'ocn%d'%apaid, + data: { + noisedb: wc.tn(chndb), + anode: wc.tn(anode), + resmp: [ + ], + }, + }, + + // In principle there may be multiple filters of each of each + // type. Define them above and collect them by type here. + local filters = { + channel: [ single, ], + group: [ ], + status: [ ], + }, + + local obnf = pg.pnode({ + + type: 'OmnibusNoiseFilter', + name: 'nf%d'%apaid, + data: { + + // Nonzero forces the number of ticks in the waveform + nticks: 0, + + // channel bin ranges are ignored + // only when the channelmask is merged to `bad` + maskmap: {sticky: "bad", ledge: "bad", noisy: "bad"}, + channel_filters: [wc.tn(f) for f in filters.channel], + grouped_filters: [wc.tn(f) for f in filters.group], + channel_status_filters: [wc.tn(f) for f in filters.status], + noisedb: wc.tn(chndb), + intraces: 'orig%d' % apaid, // frame tag get all traces + outtraces: 'raw%d' % apaid, + }, + }, uses=[chndb, anode]+filters.channel+filters.group+filters.status, nin=1, nout=1), + + res: obnf + +}.res + diff --git a/cfg/pdsp_chndb.jsonnet b/cfg/pdsp_chndb.jsonnet new file mode 100644 index 0000000..5a3bda8 --- /dev/null +++ b/cfg/pdsp_chndb.jsonnet @@ -0,0 +1,106 @@ +// This defines channel noise databases for noise filtering for pdsp. + +local wc = import "wirecell.jsonnet"; + +{ + // The "perfect noise" database is one that is free of any + // "special" considerations such as per channel variability. The + // "official" perfect chndb depends on the official "chndb-base" + // and that seems to be adulterated with specific settings. We + // try to start fresh here. + perfect(anode, fr, nsamples, tick=0.5*wc.us) :: { + local apaid = anode.data.ident, + type:'OmniChannelNoiseDB', + name: 'ocndbperfect-%s' % anode.name, + uses: [anode, fr], + data: { + anode: wc.tn(anode), + field_response: wc.tn(fr), + tick: tick, + nsamples: nsamples, + + groups: [ + std.range(apaid * 2560 + u * 40, apaid * 2560 + (u + 1) * 40 - 1) + for u in std.range(0, 19) + ] + [ + std.range(apaid * 2560 + 800 + v * 40, apaid * 2560 + 800 + (v + 1) * 40 - 1) + for v in std.range(0, 19) + ] + [ + std.range(apaid * 2560 + 1600 + w * 48, apaid * 2560 + 1600 + (w + 1) * 48 - 1) + for w in std.range(0, 19) + ], + + // last match wins + channel_info: [ + + // First entry provides default channel info across ALL + // channels. Subsequent entries override a subset of channels + // with a subset of these entries. There's no reason to + // repeat values found here in subsequent entries unless you + // wish to change them. + { + channels: std.range(apaid * 2560, (apaid + 1) * 2560 - 1), + nominal_baseline: 2350.0, // adc count + gain_correction: 1.0, // unitless + response_offset: 0.0, // ticks? + pad_window_front: 10, // ticks? + pad_window_back: 10, // ticks? + decon_limit: 0.02, + decon_limit1: 0.09, + adc_limit: 15, + roi_min_max_ratio: 0.8, // default 0.8 + min_rms_cut: 1.0, // units??? + max_rms_cut: 30.0, // units??? + + // parameter used to make "rcrc" spectrum + rcrc: 1.1 * wc.millisecond, // 1.1 for collection, 3.3 for induction + rc_layers: 1, // default 2 + + // parameters used to make "config" spectrum + reconfig: {}, + + // list to make "noise" spectrum mask + freqmasks: [], + + // field response waveform to make "response" spectrum. + response: {}, + + }, + + { + //channels: { wpid: wc.WirePlaneId(wc.Ulayer) }, + channels: std.range(apaid * 2560, apaid * 2560 + 800- 1), + /// this will use an average calculated from the anode + // response: { wpid: wc.WirePlaneId(wc.Ulayer) }, + /// this uses hard-coded waveform. + response_offset: 120, // offset of the negative peak + pad_window_front: 20, + decon_limit: 0.02, + decon_limit1: 0.07, + roi_min_max_ratio: 3.0, + }, + + { + //channels: { wpid: wc.WirePlaneId(wc.Vlayer) }, + channels: std.range(apaid * 2560 + 800, apaid * 2560 + 1600- 1), + /// this will use an average calculated from the anode + // response: { wpid: wc.WirePlaneId(wc.Vlayer) }, + /// this uses hard-coded waveform. + decon_limit: 0.01, + decon_limit1: 0.08, + roi_min_max_ratio: 1.5, + }, + + { + //channels: { wpid: wc.WirePlaneId(wc.Wlayer) }, + channels: std.range(apaid * 2560 + 1600, apaid * 2560 + 2560- 1), + nominal_baseline: 900.0, + decon_limit: 0.05, + decon_limit1: 0.08, + }, + + + ], + } // data + } // perfect() +} diff --git a/cfg/sp.jsonnet b/cfg/sp.jsonnet new file mode 100644 index 0000000..1c8d9c5 --- /dev/null +++ b/cfg/sp.jsonnet @@ -0,0 +1,84 @@ +local wc = import "wirecell.jsonnet"; +local pg = import "pgraph.jsonnet"; + + +// Signal processing. +// +// Note, spfilt are a list of filter objects which MUST match +// hard-wired names in the C++, sorry. See, eg +// pgrapher/experiment/pdsp/sp-filters.jsonnet. +function(anode, fieldresp, elecresp, spfilt, adcpermv, perchan=null) { + local apaid = anode.data.ident, + + // if perchan file name is given we need to add this to a + // "uses" list and to set the name in OmnibusSigProc. + local pcresp = { + type: "PerChannelResponse", + data: { + filename: perchan + } + }, + local pcname = if std.type(perchan) == 'null' then "" else wc.tn(pcresp), + local pcuses = if std.type(perchan) == 'null' then [] else [pcresp], + + + local sigproc = pg.pnode({ + type: 'OmnibusSigProc', + name: 'sigproc%d' % apaid, + data: { + /** + * Optimized SP parameters (May 2019) + * Associated tuning in sp-filters.jsonnet + */ + anode: wc.tn(anode), + field_response: wc.tn(fieldresp), + elecresponse: wc.tn(elecresp), + ftoffset: 0.0, // default 0.0 + ctoffset: 1.0*wc.microsecond, // default -8.0 + per_chan_resp: pcname, + fft_flag: 0, // 1 is faster but higher memory, 0 is slightly slower but lower memory + postgain: 1.0, // default 1.2 + ADC_mV: adcpermv, // 4096 / (1400.0 * wc.mV), + troi_col_th_factor: 5.0, // default 5 + troi_ind_th_factor: 3.0, // default 3 + lroi_rebin: 6, // default 6 + lroi_th_factor: 3.5, // default 3.5 + lroi_th_factor1: 0.7, // default 0.7 + lroi_jump_one_bin: 1, // default 0 + + r_th_factor: 3.0, // default 3 + r_fake_signal_low_th: 375, // default 500 + r_fake_signal_high_th: 750, // default 1000 + r_fake_signal_low_th_ind_factor: 1.0, // default 1 + r_fake_signal_high_th_ind_factor: 1.0, // default 1 + r_th_peak: 3.0, // default 3.0 + r_sep_peak: 6.0, // default 6.0 + r_low_peak_sep_threshold_pre: 1200, // default 1200 + + // frame tags + wiener_tag: 'wiener%d' % apaid, + wiener_threshold_tag: 'threshold%d' % apaid, + decon_charge_tag: 'decon_charge%d' % apaid, + gauss_tag: 'gauss%d' % apaid, + + use_roi_debug_mode: false, + tight_lf_tag: 'tight_lf%d' % apaid, + loose_lf_tag: 'loose_lf%d' % apaid, + cleanup_roi_tag: 'cleanup_roi%d' % apaid, + break_roi_loop1_tag: 'break_roi_1st%d' % apaid, + break_roi_loop2_tag: 'break_roi_2nd%d' % apaid, + shrink_roi_tag: 'shrink_roi%d' % apaid, + extend_roi_tag: 'extend_roi%d' % apaid, + + use_multi_plane_protection: false, + mp3_roi_tag: 'mp3_roi%d' % apaid, + mp2_roi_tag: 'mp2_roi%d' % apaid, + + isWrapped: false, + // process_planes: [0, 2], + } + }, nin=1, nout=1, uses=[anode, fieldresp, elecresp] + pcuses + spfilt), + + res: sigproc +}.res + diff --git a/cfg/toyzero.jsonnet b/cfg/toyzero.jsonnet index ca3721a..4fda8a0 100644 --- a/cfg/toyzero.jsonnet +++ b/cfg/toyzero.jsonnet @@ -1,11 +1,10 @@ local wc = import "wirecell.jsonnet"; local pg = import "pgraph.jsonnet"; - +local ut = import "utils.jsonnet"; // define general object for toyzero { - protodune_params: import "pgrapher/experiment/pdsp/simparams.jsonnet", // return a "wireobj" wire_file(filename) : { @@ -25,55 +24,67 @@ local pg = import "pgraph.jsonnet"; } for vol in volumes], - // Return "PlaneImpactResponse" objects. daq and elec likely come - // from params. - pirs(respf, daq, elec): { - - local field_resp = { - type: "FieldResponse", - name: respf, - data: { - filename: respf - } - }, - - local binning = { nticks: daq.nticks, tick: daq.tick }, - - local rc_resp = { - type: "RCResponse", - data: binning { - width: 1.0*wc.ms, - } - }, - - local elec_resp = { - type: "ColdElecResponse", - data: binning { - shaping: elec.shaping, - gain: elec.gain, - postgain: elec.postgain, - }, - }, + field_response(filename) :: { + type: "FieldResponse", + name: filename, + data: { filename: filename } + }, + elec_response(shaping, gain, postgain, nticks, tick=0.5*wc.us) :: { + type: "ColdElecResponse", + data: { + shaping: shaping, + gain: gain, + postgain: postgain, + nticks: nticks, + tick: tick, + }, + }, + rc_response(width, nticks, tick=0.5*wc.us) :: { + type: "RCResponse", + data: { + width: width, + nticks: nticks, + tick: tick, + } + }, + + // Return "PlaneImpactResponse" objects. + // fr is field_response object. + // srs is list of short response objects, eg elec_resp + // lrs is list of long response objects, eg rc_response + pirs(fr, srs, lrs): { ret: [ { type: "PlaneImpactResponse", name : "PIRplane%d" % plane, data : { plane: plane, - field_response: wc.tn(field_resp), - short_responses: [wc.tn(elec_resp)], + field_response: wc.tn(fr), + short_responses: [wc.tn(r) for r in srs], // this needs to be big enough for convolving FR*CE overall_short_padding: 200*wc.us, - long_responses: [wc.tn(rc_resp)], + long_responses: [wc.tn(r) for r in lrs], // this needs to be big enough to convolve RC long_padding: 1.5*wc.ms, }, - uses: [field_resp, elec_resp, rc_resp], + uses: [fr] + srs + lrs, } for plane in [0,1,2]], - }.ret, + adcpermv(adc) :: + ((1 << adc.resolution)-1) / (adc.fullscale[1]-adc.fullscale[0]), + + responses(frfile, elec, daq, width=1.0*wc.ms) :: { + + fr: $.field_response(frfile), + er: $.elec_response(elec.shaping, elec.gain, elec.postgain, + daq.nticks, daq.tick), + rc: $.rc_response(width, daq.nticks, daq.tick), + pirs: $.pirs(self.fr, [self.er], [self.rc]) + }, + + default_seeds: [0, 1, 2, 3, 4], random(seeds = $.default_seeds) : { type: "Random", @@ -105,19 +116,53 @@ local pg = import "pgraph.jsonnet"; }, nin=1, nout=1), + // A per anode configure node for simulating noise. + noisesim(anode, noisef, daq, chstat=null, rnd=$.random()) : { + local apaid = anode.data.ident, + + local noise_model = { + type: "EmpiricalNoiseModel", + name: "emperical-noise-model-%d" % apaid, + data: { + anode: wc.tn(anode), + chanstat: if std.type(chstat) == "null" then "" else wc.tn(chstat), + spectra_file: noisef, + nsamples: daq.nticks, + period: daq.tick, + wire_length_scale: 1.0*wc.cm, // optimization binning + }, + uses: [anode] + if std.type(chstat) == "null" then [] else [chstat], + }, + ret: pg.pnode({ + type: "AddNoise", + name: "addnoise-" + noise_model.name, + data: { + rng: wc.tn(rnd), + model: wc.tn(noise_model), + nsamples: daq.nticks, + replacement_percentage: 0.02, // random optimization + }}, nin=1, nout=1, uses=[rnd, noise_model]), + }.ret, + + digisim(anode, adc) :: { + local apaid = anode.data.ident, + ret: pg.pnode({ + type: "Digitizer", + name: 'Digitizer%d' % apaid, + data : adc { + anode: wc.tn(anode), + frame_tag: "orig%d"%apaid, + } + }, nin=1, nout=1, uses=[anode]), + }.ret, - // A pipeline of nodes to simulate one APA. - // - // The vol, daq, adc, lar likely comes from params. - // The noisef should be a file name. - // fixme: probably should be broken up... - apasim(anode, pirs, vol, daq, adc, lar, noisef=null, rnd=$.random()) : { + sigsim(anode, pirs, daq, lar, rnd=$.random()) : { local apaid = anode.data.ident, local ductor = pg.pnode({ type:'DepoTransform', - name:'DepoTransformt%d' % apaid, + name:'DepoTransform%d' % apaid, data: { rng: wc.tn(rnd), anode: wc.tn(anode), @@ -143,61 +188,95 @@ local pg = import "pgraph.jsonnet"; toffset: 0, nticks: daq.nticks, }, - }, nin=1, nout=1), - - local digitizer = pg.pnode({ - type: "Digitizer", - name: 'Digitizer%d' % apaid, - data : adc { - anode: wc.tn(anode), - frame_tag: "orig%d"%apaid, - } }, nin=1, nout=1, uses=[anode]), - - local csdb = null, + ret: pg.pipeline([ductor, reframer]), + }.ret, - local noise_model = { - type: "EmpiricalNoiseModel", - name: "emperical-noise-model-%d" % apaid, + // A kitchen sink pipeline of nodes to simulate one APA. + // + // The daq, adc, lar likely comes from params. + // + // Fullest chain is sig + noise -> adc + // + // If no lar or pirs is given, then no signal. + // If no noisef given, then no noise. + // If no adc given, then no digitizer + // The tier can be 'adc' or something else if no digitizer. + sim(anode, pirs, daq, adc, lar, noisef=null, tier='adc', rnd=$.random()) : { + + local apaid = anode.data.ident, + + local beg = if std.type(lar) == "null" || std.type(pirs) == "null" then [] else [ + $.sigsim(anode, pirs, daq, lar, rnd)], + + local mid = if std.type(noisef) == "null" then [] else [ + $.noisesim(anode, noisef, daq, rnd=rnd)], + + local end = if tier == 'adc' then [$.digisim(anode, adc)] else [], + + pipeline: pg.pipeline(beg + mid + end), + }.pipeline, + + + // Return a DepoSplat node + splat(anode, daq, lar, rnd=null) :: { + + local apaid = anode.data.ident, + local frame_tag = "splat%d" % apaid, + + local rextra = if std.type(rnd) == "null" + then { data:{}, uses:[] } + else { + data: { fluctuate:true, rng: wc.tn(rnd) }, + uses: [rnd] + }, + + local splat = pg.pnode({ + type: "DepoSplat", + name: "splat%d"%apaid, data: { + frame_tag: frame_tag, anode: wc.tn(anode), - chanstat: if std.type(csdb) == "null" then "" else wc.tn(csdb), - spectra_file: noisef, - nsamples: daq.nticks, - period: daq.tick, - wire_length_scale: 1.0*wc.cm, // optimization binning - }, - uses: [anode] + if std.type(csdb) == "null" then [] else [csdb], - }, - local noise = pg.pnode({ - type: "AddNoise", - name: "addnoise-" + noise_model.name, + continuous: false, + fixed: true, + drift_speed: lar.drift_speed, + readout_time: daq.nticks*daq.tick, + start_time: 0, + tick: daq.tick, + nsigma: 3, + } + rextra.data + }, nin=1, nout=1, uses=[anode]+rextra.uses), + + local reframer = pg.pnode({ + type: 'Reframer', + name: 'Reframer%d' % apaid, data: { - rng: wc.tn(rnd), - model: wc.tn(noise_model), - nsamples: daq.nticks, - replacement_percentage: 0.02, // random optimization - }}, nin=1, nout=1, uses=[rnd, noise_model]), - - local beg = [ductor, reframer], + anode: wc.tn(anode), + tags: [], + frame_tag: frame_tag, + fill: 0.0, + tbin: 0, + toffset: 0, + nticks: daq.nticks, + }, + }, nin=1, nout=1, uses=[anode]), - local mid = if std.type(noisef) == "null" then [] else [noise], + ret: pg.pipeline([splat, reframer]), + }.ret, - local end = [digitizer], - pipeline: pg.pipeline(beg + mid + end), - }.pipeline, + // top-level stuff local plugins = [ "WireCellSio", "WireCellAux", "WireCellGen", "WireCellSigProc", - "WireCellApps", "WireCellPgraph"], + "WireCellApps", "WireCellPgraph", "WireCellTbb"], - main(graph) :: { - local app = { - type: 'Pgrapher', + main(graph, app) :: { + local appcfg = { + type: app, data: { edges: pg.edges(graph) }, @@ -206,9 +285,9 @@ local pg = import "pgraph.jsonnet"; type: "wire-cell", data: { plugins: plugins, - apps: ["Pgrapher"], + apps: [appcfg.type] } }, - seq: [cmdline] + pg.uses(graph) + [app], + seq: [cmdline] + pg.uses(graph) + [appcfg], }.seq } diff --git a/cfg/utils.jsonnet b/cfg/utils.jsonnet new file mode 100644 index 0000000..e715b2c --- /dev/null +++ b/cfg/utils.jsonnet @@ -0,0 +1,23 @@ +// fixme: many/all of these probably could get lifted to wct/cfg/ + +{ + // Return true if string s has at least one % symbol. + haspct(s, pct="%"):: std.length(std.findSubstr(pct, s))>0, + + stripext(fn, n=1) :: { + local parts = std.split(fn, "."), + local siz = std.length(parts), + ret: std.join(".", parts[:siz+1-n]) + }.ret, + + basename(fn) :: { + local parts = std.split(fn, "/"), + ret: parts[std.length(parts)-1] + }.ret, + + // append a suffix to the base name of a file name, prior to .ext + basename_append(filename, suffix) :: { + local l = std.split(filename, "."), + ret:"%s%s.%s"%[l[0], suffix, l[1]] + }.ret, +} diff --git a/dot.envrc b/dot.envrc index 361e981..1798950 100644 --- a/dot.envrc +++ b/dot.envrc @@ -1,5 +1,6 @@ -load_prefix $HOME/dev/ls4gan/toyzero/wire-cell-toolkit/install -path_add WIRECELL_PATH $HOME/dev/ls4gan/toyzero/wire-cell-toolkit/cfg +load_prefix $HOME/wrk/ls4gan/toyzero/wire-cell-toolkit/install +path_add WIRECELL_PATH $HOME/wrk/ls4gan/toyzero/wire-cell-toolkit/cfg +path_add WIRECELL_PATH $HOME/wrk/ls4gan/toyzero/wire-cell-data load_prefix $HOME/opt/jsonnet layout python3 diff --git a/index.html b/index.html index e39dfc5..d2b8e5c 100644 --- a/index.html +++ b/index.html @@ -3,7 +3,7 @@ "http://www.w3.org/TR/xhtml1/DTD/xhtml1-strict.dtd"> - + LS4GAN's Toy Zero @@ -257,7 +257,7 @@

Table of Contents

-The toyzero package provides first steps toward LS4GAN. +The toyzero package provides first steps toward LS4GAN. See also next-steps.html

@@ -334,7 +334,7 @@

2.1 Environment

A one time setup of direnv:

-
+
 $ cd toyzero
 $ cp dot.envrc .envrc
 $ emacs .envrc  # edit to taste
@@ -352,7 +352,7 @@ 

2.2 Check Wire-Cell Toolki Check if WCT is available with these commands:

-
+
 $ wire-cell --help
 $ wcsonnet wirecell.jsonnet
 $ echo $WIRECELL_PATH
@@ -374,7 +374,7 @@ 

2.3 Python

various other Python 3 packages are required.

-
+
 $ cd toyzero
 $ pip install -r requirements.txt
 
@@ -383,7 +383,7 @@

2.3 Python

Some checks:

-
+
 $ wirecell-<TAB>
 $ wirecell-util --help
 $ snakemake --help
@@ -406,7 +406,7 @@ 

3 Usage

need run only:

-
+
 $ snakemake -jall all
 
@@ -415,7 +415,7 @@

3 Usage

To see what is produced:

-
+
 $ tree data plots
 
@@ -424,7 +424,7 @@

3 Usage

you can do:

-
+
 $ snakemake -jall just_images
 
@@ -435,7 +435,7 @@

3 Usage

may run as:

-
+
 $ snakemake -jall just_images --notemp
 
@@ -460,7 +460,7 @@

4 Details

manner:

-
+
 $ snakemake -jall all_resp
 $ snakemake -jall all_wires
 $ snakemake -jall all_depos
@@ -506,7 +506,7 @@ 

4.1 resp

-
+

fake-resps-diagnostic.png

@@ -516,7 +516,7 @@

4.1 resp

-
+

real-resps-diagnostic.png

@@ -549,7 +549,7 @@

4.3 depos

for input to the WCT simulation. It produces a file:

-
+
 data/depos/depos.npz
 
@@ -557,7 +557,7 @@

4.3 depos

The file is in Numpy format with three types of arrays named like:

-
+
 depo_data_<N>
 depo_info_<N>
 track_info_<N>
@@ -590,7 +590,7 @@ 

4.3 depos

-
+

depos-diagnostic.png

@@ -624,9 +624,18 @@

4.4 frames

Wire-Cell simulation. The "frame" file format is described elsewhere. For here, we treat it as a temporary.

+ +

+The simulation is internally structured as a DAG as shown: +

+ + +
+

dag.png +

+
-

4.5 images

@@ -638,7 +647,7 @@

4.5 images

match. For example:

-
+
 ❯ wirecell-util npzls data/images/real/protodune-orig-0-1-W.npz 
 protodune-orig-0-1-W (960, 6000)
 
@@ -698,7 +707,7 @@

5.1 Image formats

-
+

frames-fake-apa0.png

@@ -708,7 +717,7 @@

5.1 Image formats

-
+

frames-fake-apa0.png

@@ -717,7 +726,7 @@

5.1 Image formats

If you need more zoom from your evince PDF viewer try

-
+
 ❯ gsettings set org.gnome.Evince page-cache-size 2014
 
@@ -747,7 +756,7 @@

5.2 Color maps

as command line options:

-
+
 ❯ wirecell-util npz-to-img --help
 Usage: wirecell-util npz-to-img [OPTIONS] NPZFILE
 
@@ -782,7 +791,7 @@ 

5.2 Color maps

-
+

protodune-orig-0-2-U.png

@@ -792,7 +801,7 @@

5.2 Color maps

-
+

protodune-orig-0-2-U.png

@@ -802,7 +811,7 @@

5.2 Color maps

-
+

protodune-orig-0-2-U.png

@@ -812,7 +821,7 @@

5.2 Color maps

-
+

protodune-orig-0-2-U.png

@@ -833,7 +842,7 @@

5.3 Real vs Fake

-
+

Sorry, your browser does not support SVG.

@@ -845,7 +854,7 @@

5.3 Real vs Fake

-
+

Sorry, your browser does not support SVG.

@@ -891,7 +900,7 @@

6 Configuration

To customize,

-
+
 $ cp toyzero.yaml mycfg.yaml
 $ emacs mycfg.yaml
 $ snakemake -jall --configfile mycfg.yaml all
@@ -907,7 +916,7 @@ 

6 Configuration

You may also customize specific parameters from the command line:

-
+
 $ snakemake -jall --config outdir=junk ntracks=10 all_depos
 $ eom junk/plots/depos-diagnostic.png 
 
@@ -948,7 +957,7 @@

7 Snakemake

-
+

Sorry, your browser does not support SVG.

@@ -974,7 +983,7 @@

8 Hacking

so like:

-
+
 $ git clone git@github.com:wirecell/wire-cell-python.git
 $ cd wire-cell-python/
 $ pip install -U -e .
@@ -990,7 +999,7 @@ 

8 Hacking

Author: BV

-

Created: 2021-07-20 Tue 10:49

+

Created: 2021-07-21 Wed 15:18

Validate

diff --git a/testit.sh b/testit.sh new file mode 100755 index 0000000..abdda7e --- /dev/null +++ b/testit.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +set -x + +mkdir -p testit + +tiers=(orig gauss) +for domain in real fake +do + taps="" + for tier in ${tiers[*]} + do + taps+="{\"$tier\":\"testit/${domain}-frames-${tier}.npz\"}" + done + + if [ -f testit/depos.npz ] ; then + echo "depos already generated" + else + wirecell-gen depo-lines \ + --seed 1234 \ + --tracks 100 \ + --sets 1 \ + --diagonal '16000.0*mm,6100.0*mm,7000.0*mm' \ + --corner '-8000.0*mm,0.0*mm,0.0*mm' \ + --output testit/depos.npz || exit + fi + + if [ -f testit/${domain}-frames-orig-apa0.npz ] ; then + echo "wire-cell has run already for $domain" + else + wire-cell \ + -l stdout -L debug -P cfg\ + -A input=testit/depos.npz \ + --tla-code taps=$taps \ + -A wires=data/wires/wires.json.bz2 \ + -A resps_sim=data/resps/${domain}-resps.json.bz2 \ + -A resps_sigproc=data/resps/fake-resps.json.bz2 \ + -A thread=multi \ + -A noisef=protodune-noise-spectra-v1.json.bz2 \ + -c cfg/main-depos-sigproc.jsonnet || exit + fi + + + for tier in orig gauss + do + for apaid in 0 1 2 3 4 5 + do + tag=${tier}${apaid} + wirecell-gen \ + plot-sim \ + testit/${domain}-frames-${tier}-apa${apaid}.npz \ + -p frames -b 0,2560 --tag ${tier}${apaid} \ + testit/${domain}-frames-${tier}-apa${apaid}.png || exit + done + done +done + + + diff --git a/toyzero-all-dag.dot b/toyzero-all-dag.dot index ba373d9..3c12237 100644 --- a/toyzero-all-dag.dot +++ b/toyzero-all-dag.dot @@ -2,214 +2,214 @@ digraph snakemake_dag { graph[bgcolor=white, margin=0]; node[shape=box, style=rounded, fontname=sans, fontsize=10, penwidth=2]; edge[penwidth=2, color=grey]; - 0[label = "just_images", color = "0.17 0.6 0.85", style="rounded"]; - 1[label = "split_images", color = "0.37 0.6 0.85", style="rounded"]; - 2[label = "sim_frames\ndomain: real", color = "0.60 0.6 0.85", style="rounded"]; - 3[label = "get_wires", color = "0.43 0.6 0.85", style="rounded"]; - 4[label = "get_resp_real", color = "0.13 0.6 0.85", style="rounded"]; - 5[label = "gen_depos", color = "0.23 0.6 0.85", style="rounded"]; - 6[label = "split_images", color = "0.37 0.6 0.85", style="rounded"]; - 7[label = "sim_frames\ndomain: fake", color = "0.60 0.6 0.85", style="rounded"]; - 8[label = "gen_resp_fake", color = "0.30 0.6 0.85", style="rounded"]; - 9[label = "all", color = "0.27 0.6 0.85", style="rounded"]; - 10[label = "plot_resp\ndomain: real", color = "0.57 0.6 0.85", style="rounded"]; - 11[label = "plot_resp\ndomain: fake", color = "0.57 0.6 0.85", style="rounded"]; - 12[label = "plot_wires", color = "0.07 0.6 0.85", style="rounded"]; - 13[label = "plot_depos", color = "0.40 0.6 0.85", style="rounded"]; - 14[label = "plot_frames\napa: 0\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 15[label = "plot_frames\napa: 1\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 16[label = "plot_frames\napa: 2\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 17[label = "plot_frames\napa: 3\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 18[label = "plot_frames\napa: 4\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 19[label = "plot_frames\napa: 5\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 20[label = "plot_frames\napa: 0\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 21[label = "plot_frames\napa: 1\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 22[label = "plot_frames\napa: 2\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 23[label = "plot_frames\napa: 3\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 24[label = "plot_frames\napa: 4\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 25[label = "plot_frames\napa: 5\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 26[label = "plot_frames\napa: 0\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 27[label = "plot_frames\napa: 1\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 28[label = "plot_frames\napa: 2\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 29[label = "plot_frames\napa: 3\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 30[label = "plot_frames\napa: 4\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 31[label = "plot_frames\napa: 5\next: png", color = "0.00 0.6 0.85", style="rounded"]; - 32[label = "plot_frames\napa: 0\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 33[label = "plot_frames\napa: 1\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 34[label = "plot_frames\napa: 2\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 35[label = "plot_frames\napa: 3\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 36[label = "plot_frames\napa: 4\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 37[label = "plot_frames\napa: 5\next: pdf", color = "0.00 0.6 0.85", style="rounded"]; - 38[label = "plot_frames_hidpi", color = "0.33 0.6 0.85", style="rounded"]; - 39[label = "plot_frames_hidpi", color = "0.33 0.6 0.85", style="rounded"]; - 40[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 41[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 42[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 43[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 44[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 45[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 46[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 47[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 48[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 49[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 50[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 51[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 52[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 53[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 54[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 55[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 56[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 57[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 58[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 59[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 60[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 61[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 62[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 63[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 64[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 65[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 66[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 67[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 68[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; - 69[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 0[label = "all", color = "0.38 0.6 0.85", style="rounded"]; + 1[label = "get_resp_real", color = "0.00 0.6 0.85", style="rounded"]; + 2[label = "gen_resp_fake", color = "0.03 0.6 0.85", style="rounded"]; + 3[label = "plot_resp\ndomain: real", color = "0.60 0.6 0.85", style="rounded"]; + 4[label = "plot_resp\ndomain: fake", color = "0.60 0.6 0.85", style="rounded"]; + 5[label = "get_wires", color = "0.54 0.6 0.85", style="rounded"]; + 6[label = "plot_wires", color = "0.48 0.6 0.85", style="rounded"]; + 7[label = "gen_depos", color = "0.06 0.6 0.85", style="rounded"]; + 8[label = "plot_depos", color = "0.25 0.6 0.85", style="rounded"]; + 9[label = "sim_frames\ndomain: real", color = "0.19 0.6 0.85", style="rounded"]; + 10[label = "sim_frames\ndomain: fake", color = "0.19 0.6 0.85", style="rounded"]; + 11[label = "plot_frames\napa: 0\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 12[label = "plot_frames\napa: 1\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 13[label = "plot_frames\napa: 2\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 14[label = "plot_frames\napa: 3\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 15[label = "plot_frames\napa: 4\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 16[label = "plot_frames\napa: 5\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 17[label = "plot_frames\napa: 0\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 18[label = "plot_frames\napa: 1\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 19[label = "plot_frames\napa: 2\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 20[label = "plot_frames\napa: 3\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 21[label = "plot_frames\napa: 4\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 22[label = "plot_frames\napa: 5\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 23[label = "plot_frames\napa: 0\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 24[label = "plot_frames\napa: 1\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 25[label = "plot_frames\napa: 2\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 26[label = "plot_frames\napa: 3\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 27[label = "plot_frames\napa: 4\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 28[label = "plot_frames\napa: 5\next: png", color = "0.35 0.6 0.85", style="rounded"]; + 29[label = "plot_frames\napa: 0\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 30[label = "plot_frames\napa: 1\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 31[label = "plot_frames\napa: 2\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 32[label = "plot_frames\napa: 3\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 33[label = "plot_frames\napa: 4\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 34[label = "plot_frames\napa: 5\next: pdf", color = "0.35 0.6 0.85", style="rounded"]; + 35[label = "plot_frames_hidpi", color = "0.10 0.6 0.85", style="rounded"]; + 36[label = "plot_frames_hidpi", color = "0.10 0.6 0.85", style="rounded"]; + 37[label = "split_images", color = "0.22 0.6 0.85", style="rounded"]; + 38[label = "split_images", color = "0.22 0.6 0.85", style="rounded"]; + 39[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 40[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 41[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 42[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 43[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 44[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 45[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 46[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 47[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 48[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 49[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 50[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 51[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 52[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 53[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 54[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 55[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 56[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 57[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 58[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: png\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 59[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 60[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 61[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 62[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 63[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: pdf\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 64[label = "plot_split_images\napa: 2\ncmap: seismic\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 65[label = "plot_split_images\napa: 2\ncmap: Spectral\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 66[label = "plot_split_images\napa: 2\ncmap: terrain\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 67[label = "plot_split_images\napa: 2\ncmap: coolwarm\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 68[label = "plot_split_images\napa: 2\ncmap: viridis\nevent: 0\next: svg\nplane: U", color = "0.63 0.6 0.85", style="rounded"]; + 69[label = "just_images", color = "0.29 0.6 0.85", style="rounded"]; 1 -> 0 + 2 -> 0 + 3 -> 0 + 4 -> 0 + 5 -> 0 6 -> 0 - 2 -> 1 - 3 -> 2 - 4 -> 2 - 5 -> 2 - 3 -> 5 - 7 -> 6 - 3 -> 7 - 8 -> 7 + 7 -> 0 + 8 -> 0 + 9 -> 0 + 10 -> 0 + 11 -> 0 + 12 -> 0 + 13 -> 0 + 14 -> 0 + 15 -> 0 + 16 -> 0 + 17 -> 0 + 18 -> 0 + 19 -> 0 + 20 -> 0 + 21 -> 0 + 22 -> 0 + 23 -> 0 + 24 -> 0 + 25 -> 0 + 26 -> 0 + 27 -> 0 + 28 -> 0 + 29 -> 0 + 30 -> 0 + 31 -> 0 + 32 -> 0 + 33 -> 0 + 34 -> 0 + 35 -> 0 + 36 -> 0 + 37 -> 0 + 38 -> 0 + 39 -> 0 + 40 -> 0 + 41 -> 0 + 42 -> 0 + 43 -> 0 + 44 -> 0 + 45 -> 0 + 46 -> 0 + 47 -> 0 + 48 -> 0 + 49 -> 0 + 50 -> 0 + 51 -> 0 + 52 -> 0 + 53 -> 0 + 54 -> 0 + 55 -> 0 + 56 -> 0 + 57 -> 0 + 58 -> 0 + 59 -> 0 + 60 -> 0 + 61 -> 0 + 62 -> 0 + 63 -> 0 + 64 -> 0 + 65 -> 0 + 66 -> 0 + 67 -> 0 + 68 -> 0 + 1 -> 2 + 1 -> 3 + 2 -> 4 + 5 -> 6 5 -> 7 - 4 -> 8 - 4 -> 9 - 8 -> 9 - 10 -> 9 - 11 -> 9 - 3 -> 9 - 12 -> 9 + 7 -> 8 5 -> 9 - 13 -> 9 - 2 -> 9 - 7 -> 9 - 14 -> 9 - 15 -> 9 - 16 -> 9 - 17 -> 9 - 18 -> 9 - 19 -> 9 - 20 -> 9 - 21 -> 9 - 22 -> 9 - 23 -> 9 - 24 -> 9 - 25 -> 9 - 26 -> 9 - 27 -> 9 - 28 -> 9 - 29 -> 9 - 30 -> 9 - 31 -> 9 - 32 -> 9 - 33 -> 9 - 34 -> 9 - 35 -> 9 - 36 -> 9 - 37 -> 9 - 38 -> 9 - 39 -> 9 1 -> 9 - 6 -> 9 - 40 -> 9 - 41 -> 9 - 42 -> 9 - 43 -> 9 - 44 -> 9 - 45 -> 9 - 46 -> 9 - 47 -> 9 - 48 -> 9 - 49 -> 9 - 50 -> 9 - 51 -> 9 - 52 -> 9 - 53 -> 9 - 54 -> 9 - 55 -> 9 - 56 -> 9 - 57 -> 9 - 58 -> 9 - 59 -> 9 - 60 -> 9 - 61 -> 9 - 62 -> 9 - 63 -> 9 - 64 -> 9 - 65 -> 9 - 66 -> 9 - 67 -> 9 - 68 -> 9 - 69 -> 9 - 4 -> 10 - 8 -> 11 - 3 -> 12 - 5 -> 13 - 2 -> 14 - 2 -> 15 - 2 -> 16 - 2 -> 17 - 2 -> 18 - 2 -> 19 - 2 -> 20 - 2 -> 21 - 2 -> 22 - 2 -> 23 - 2 -> 24 - 2 -> 25 - 7 -> 26 - 7 -> 27 - 7 -> 28 - 7 -> 29 - 7 -> 30 - 7 -> 31 - 7 -> 32 - 7 -> 33 - 7 -> 34 - 7 -> 35 - 7 -> 36 - 7 -> 37 - 20 -> 38 - 32 -> 39 - 1 -> 40 - 1 -> 41 - 1 -> 42 - 1 -> 43 - 1 -> 44 - 1 -> 45 - 1 -> 46 - 1 -> 47 - 1 -> 48 - 1 -> 49 - 1 -> 50 - 1 -> 51 - 1 -> 52 - 1 -> 53 - 1 -> 54 - 6 -> 55 - 6 -> 56 - 6 -> 57 - 6 -> 58 - 6 -> 59 - 6 -> 60 - 6 -> 61 - 6 -> 62 - 6 -> 63 - 6 -> 64 - 6 -> 65 - 6 -> 66 - 6 -> 67 - 6 -> 68 - 6 -> 69 + 7 -> 9 + 5 -> 10 + 2 -> 10 + 7 -> 10 + 9 -> 11 + 9 -> 12 + 9 -> 13 + 9 -> 14 + 9 -> 15 + 9 -> 16 + 9 -> 17 + 9 -> 18 + 9 -> 19 + 9 -> 20 + 9 -> 21 + 9 -> 22 + 10 -> 23 + 10 -> 24 + 10 -> 25 + 10 -> 26 + 10 -> 27 + 10 -> 28 + 10 -> 29 + 10 -> 30 + 10 -> 31 + 10 -> 32 + 10 -> 33 + 10 -> 34 + 17 -> 35 + 29 -> 36 + 9 -> 37 + 10 -> 38 + 37 -> 39 + 37 -> 40 + 37 -> 41 + 37 -> 42 + 37 -> 43 + 37 -> 44 + 37 -> 45 + 37 -> 46 + 37 -> 47 + 37 -> 48 + 37 -> 49 + 37 -> 50 + 37 -> 51 + 37 -> 52 + 37 -> 53 + 38 -> 54 + 38 -> 55 + 38 -> 56 + 38 -> 57 + 38 -> 58 + 38 -> 59 + 38 -> 60 + 38 -> 61 + 38 -> 62 + 38 -> 63 + 38 -> 64 + 38 -> 65 + 38 -> 66 + 38 -> 67 + 38 -> 68 + 37 -> 69 + 38 -> 69 } diff --git a/toyzero-all-dag.pdf b/toyzero-all-dag.pdf index 3673f63..17ad384 100644 Binary files a/toyzero-all-dag.pdf and b/toyzero-all-dag.pdf differ diff --git a/toyzero-all-dag.svg b/toyzero-all-dag.svg index 4255248..2e349af 100644 --- a/toyzero-all-dag.svg +++ b/toyzero-all-dag.svg @@ -4,1472 +4,1472 @@ - + snakemake_dag - + 0 - -just_images + +all 1 - -split_images + +get_resp_real 1->0 - - + + + + + +2 + +gen_resp_fake + + + +1->2 + + + + + +3 + +plot_resp +domain: real + + + +1->3 + + 9 - -all + +sim_frames +domain: real - + 1->9 - - + + - - -40 - -plot_split_images -apa: 2 -cmap: seismic -event: 0 -ext: png -plane: U - - - -1->40 - - + + +2->0 + + - - -41 - -plot_split_images -apa: 2 -cmap: Spectral -event: 0 -ext: png -plane: U - - - -1->41 - - + + +4 + +plot_resp +domain: fake - - -42 - -plot_split_images -apa: 2 -cmap: terrain -event: 0 -ext: png -plane: U - - - -1->42 - - + + +2->4 + + - - -43 - -plot_split_images -apa: 2 -cmap: coolwarm -event: 0 -ext: png -plane: U - - - -1->43 - - + + +10 + +sim_frames +domain: fake - - -44 - -plot_split_images -apa: 2 -cmap: viridis -event: 0 -ext: png -plane: U - - - -1->44 - - + + +2->10 + + - - -45 - -plot_split_images -apa: 2 -cmap: seismic -event: 0 -ext: pdf -plane: U - - - -1->45 - - + + +3->0 + + - - -46 - -plot_split_images -apa: 2 -cmap: Spectral -event: 0 -ext: pdf -plane: U - - - -1->46 - - + + +4->0 + + - - -47 - -plot_split_images -apa: 2 -cmap: terrain -event: 0 -ext: pdf -plane: U - - - -1->47 - - + + +5 + +get_wires - - -48 - -plot_split_images -apa: 2 -cmap: coolwarm -event: 0 -ext: pdf -plane: U - - - -1->48 - - + + +5->0 + + - - -49 - -plot_split_images -apa: 2 -cmap: viridis -event: 0 -ext: pdf -plane: U - - - -1->49 - - + + +6 + +plot_wires - - -50 - -plot_split_images -apa: 2 -cmap: seismic -event: 0 -ext: svg -plane: U - - - -1->50 - - + + +5->6 + + - - -51 - -plot_split_images -apa: 2 -cmap: Spectral -event: 0 -ext: svg -plane: U - - - -1->51 - - + + +7 + +gen_depos - - -52 - -plot_split_images -apa: 2 -cmap: terrain -event: 0 -ext: svg -plane: U - - - -1->52 - - + + +5->7 + + - - -53 - -plot_split_images -apa: 2 -cmap: coolwarm -event: 0 -ext: svg -plane: U - - - -1->53 - - + + +5->9 + + - - -54 - -plot_split_images -apa: 2 -cmap: viridis -event: 0 -ext: svg -plane: U - - - -1->54 - - + + +5->10 + + - - -2 - -sim_frames -domain: real + + +6->0 + + - - -2->1 - - + + +7->0 + + - - -2->9 - - + + +8 + +plot_depos + + + +7->8 + + + + + +7->9 + + + + + +7->10 + + + + + +8->0 + + + + + +9->0 + + + + + +11 + +plot_frames +apa: 0 +ext: png + + + +9->11 + + + + + +12 + +plot_frames +apa: 1 +ext: png + + + +9->12 + + + + + +13 + +plot_frames +apa: 2 +ext: png + + + +9->13 + + 14 - -plot_frames -apa: 0 -ext: png + +plot_frames +apa: 3 +ext: png - - -2->14 - - + + +9->14 + + 15 - -plot_frames -apa: 1 -ext: png + +plot_frames +apa: 4 +ext: png - - -2->15 - - + + +9->15 + + 16 - -plot_frames -apa: 2 -ext: png + +plot_frames +apa: 5 +ext: png - - -2->16 - - + + +9->16 + + 17 - -plot_frames -apa: 3 -ext: png + +plot_frames +apa: 0 +ext: pdf - - -2->17 - - + + +9->17 + + 18 - -plot_frames -apa: 4 -ext: png + +plot_frames +apa: 1 +ext: pdf - - -2->18 - - + + +9->18 + + 19 - -plot_frames -apa: 5 -ext: png + +plot_frames +apa: 2 +ext: pdf - - -2->19 - - + + +9->19 + + 20 - -plot_frames -apa: 0 -ext: pdf + +plot_frames +apa: 3 +ext: pdf - - -2->20 - - + + +9->20 + + 21 - -plot_frames -apa: 1 -ext: pdf + +plot_frames +apa: 4 +ext: pdf - - -2->21 - - + + +9->21 + + 22 - -plot_frames -apa: 2 -ext: pdf + +plot_frames +apa: 5 +ext: pdf - - -2->22 - - + + +9->22 + + + + + +37 + +split_images + + + +9->37 + + + + + +10->0 + + 23 - -plot_frames -apa: 3 -ext: pdf + +plot_frames +apa: 0 +ext: png - - -2->23 - - + + +10->23 + + 24 - -plot_frames -apa: 4 -ext: pdf + +plot_frames +apa: 1 +ext: png - - -2->24 - - + + +10->24 + + 25 - -plot_frames -apa: 5 -ext: pdf - - - -2->25 - - - - - -3 - -get_wires - - - -3->2 - - - - - -5 - -gen_depos - - - -3->5 - - - - - -7 - -sim_frames -domain: fake - - - -3->7 - - - - - -3->9 - - - - - -12 - -plot_wires - - - -3->12 - - - - - -4 - -get_resp_real - - - -4->2 - - + +plot_frames +apa: 2 +ext: png - - -8 - -gen_resp_fake - - - -4->8 - - - - - -4->9 - - - - - -10 - -plot_resp -domain: real - - - -4->10 - - - - - -5->2 - - - - - -5->7 - - - - - -5->9 - - - - - -13 - -plot_depos - - - -5->13 - - - - - -6 - -split_images - - - -6->0 - - - - - -6->9 - - - - - -55 - -plot_split_images -apa: 2 -cmap: seismic -event: 0 -ext: png -plane: U - - - -6->55 - - - - - -56 - -plot_split_images -apa: 2 -cmap: Spectral -event: 0 -ext: png -plane: U - - - -6->56 - - - - - -57 - -plot_split_images -apa: 2 -cmap: terrain -event: 0 -ext: png -plane: U - - - -6->57 - - - - - -58 - -plot_split_images -apa: 2 -cmap: coolwarm -event: 0 -ext: png -plane: U - - - -6->58 - - - - - -59 - -plot_split_images -apa: 2 -cmap: viridis -event: 0 -ext: png -plane: U - - - -6->59 - - - - - -60 - -plot_split_images -apa: 2 -cmap: seismic -event: 0 -ext: pdf -plane: U - - - -6->60 - - - - - -61 - -plot_split_images -apa: 2 -cmap: Spectral -event: 0 -ext: pdf -plane: U - - - -6->61 - - - - - -62 - -plot_split_images -apa: 2 -cmap: terrain -event: 0 -ext: pdf -plane: U - - - -6->62 - - - - - -63 - -plot_split_images -apa: 2 -cmap: coolwarm -event: 0 -ext: pdf -plane: U - - - -6->63 - - - - - -64 - -plot_split_images -apa: 2 -cmap: viridis -event: 0 -ext: pdf -plane: U - - - -6->64 - - - - - -65 - -plot_split_images -apa: 2 -cmap: seismic -event: 0 -ext: svg -plane: U - - - -6->65 - - - - - -66 - -plot_split_images -apa: 2 -cmap: Spectral -event: 0 -ext: svg -plane: U - - - -6->66 - - - - - -67 - -plot_split_images -apa: 2 -cmap: terrain -event: 0 -ext: svg -plane: U - - - -6->67 - - - - - -68 - -plot_split_images -apa: 2 -cmap: coolwarm -event: 0 -ext: svg -plane: U - - - -6->68 - - - - - -69 - -plot_split_images -apa: 2 -cmap: viridis -event: 0 -ext: svg -plane: U - - - -6->69 - - - - - -7->6 - - - - - -7->9 - - + + +10->25 + + 26 - -plot_frames -apa: 0 -ext: png + +plot_frames +apa: 3 +ext: png - - -7->26 - - + + +10->26 + + 27 - -plot_frames -apa: 1 -ext: png + +plot_frames +apa: 4 +ext: png - - -7->27 - - + + +10->27 + + 28 - -plot_frames -apa: 2 -ext: png + +plot_frames +apa: 5 +ext: png - - -7->28 - - + + +10->28 + + 29 - -plot_frames -apa: 3 -ext: png + +plot_frames +apa: 0 +ext: pdf - - -7->29 - - + + +10->29 + + 30 - -plot_frames -apa: 4 -ext: png + +plot_frames +apa: 1 +ext: pdf - - -7->30 - - + + +10->30 + + 31 - -plot_frames -apa: 5 -ext: png + +plot_frames +apa: 2 +ext: pdf - - -7->31 - - + + +10->31 + + 32 - -plot_frames -apa: 0 -ext: pdf + +plot_frames +apa: 3 +ext: pdf - - -7->32 - - + + +10->32 + + 33 - -plot_frames -apa: 1 -ext: pdf + +plot_frames +apa: 4 +ext: pdf - - -7->33 - - + + +10->33 + + 34 - -plot_frames -apa: 2 -ext: pdf - - - -7->34 - - - - - -35 - -plot_frames -apa: 3 -ext: pdf - - - -7->35 - - - - - -36 - -plot_frames -apa: 4 -ext: pdf + +plot_frames +apa: 5 +ext: pdf - - -7->36 - - + + +10->34 + + - - -37 - -plot_frames -apa: 5 -ext: pdf + + +38 + +split_images - + -7->37 - - +10->38 + + - - -8->7 - - + + +11->0 + + - - -8->9 - - + + +12->0 + + - - -11 - -plot_resp -domain: fake + + +13->0 + + - - -8->11 - - + + +14->0 + + - + -10->9 - - +15->0 + + - + -11->9 - - +16->0 + + + + + +17->0 + + + + + +35 + +plot_frames_hidpi - + + +17->35 + + + + -12->9 - - +18->0 + + + + + +19->0 + + - + -13->9 - - +20->0 + + + + + +21->0 + + + + + +22->0 + + - + -14->9 - - +23->0 + + - + -15->9 - - +24->0 + + - + -16->9 - - +25->0 + + - + -17->9 - - +26->0 + + - + -18->9 - - +27->0 + + - + -19->9 - - +28->0 + + - + -20->9 - - +29->0 + + - - -38 - -plot_frames_hidpi + + +36 + +plot_frames_hidpi - - -20->38 - - + + +29->36 + + - + -21->9 - - +30->0 + + - + -22->9 - - +31->0 + + - + -23->9 - - +32->0 + + - + -24->9 - - +33->0 + + - + -25->9 - - +34->0 + + - + -26->9 - - +35->0 + + - + -27->9 - - +36->0 + + - + -28->9 - - +37->0 + + + + + +39 + +plot_split_images +apa: 2 +cmap: seismic +event: 0 +ext: png +plane: U + + + +37->39 + + + + + +40 + +plot_split_images +apa: 2 +cmap: Spectral +event: 0 +ext: png +plane: U + + + +37->40 + + + + + +41 + +plot_split_images +apa: 2 +cmap: terrain +event: 0 +ext: png +plane: U + + + +37->41 + + + + + +42 + +plot_split_images +apa: 2 +cmap: coolwarm +event: 0 +ext: png +plane: U + + + +37->42 + + + + + +43 + +plot_split_images +apa: 2 +cmap: viridis +event: 0 +ext: png +plane: U + + + +37->43 + + + + + +44 + +plot_split_images +apa: 2 +cmap: seismic +event: 0 +ext: pdf +plane: U + + + +37->44 + + + + + +45 + +plot_split_images +apa: 2 +cmap: Spectral +event: 0 +ext: pdf +plane: U + + + +37->45 + + + + + +46 + +plot_split_images +apa: 2 +cmap: terrain +event: 0 +ext: pdf +plane: U + + + +37->46 + + + + + +47 + +plot_split_images +apa: 2 +cmap: coolwarm +event: 0 +ext: pdf +plane: U + + + +37->47 + + + + + +48 + +plot_split_images +apa: 2 +cmap: viridis +event: 0 +ext: pdf +plane: U + + + +37->48 + + + + + +49 + +plot_split_images +apa: 2 +cmap: seismic +event: 0 +ext: svg +plane: U + + + +37->49 + + + + + +50 + +plot_split_images +apa: 2 +cmap: Spectral +event: 0 +ext: svg +plane: U + + + +37->50 + + + + + +51 + +plot_split_images +apa: 2 +cmap: terrain +event: 0 +ext: svg +plane: U + + + +37->51 + + + + + +52 + +plot_split_images +apa: 2 +cmap: coolwarm +event: 0 +ext: svg +plane: U + + + +37->52 + + + + + +53 + +plot_split_images +apa: 2 +cmap: viridis +event: 0 +ext: svg +plane: U + + + +37->53 + + + + + +69 + +just_images + + + +37->69 + + - + -29->9 - - +38->0 + + + + + +54 + +plot_split_images +apa: 2 +cmap: seismic +event: 0 +ext: png +plane: U + + + +38->54 + + + + + +55 + +plot_split_images +apa: 2 +cmap: Spectral +event: 0 +ext: png +plane: U + + + +38->55 + + + + + +56 + +plot_split_images +apa: 2 +cmap: terrain +event: 0 +ext: png +plane: U + + + +38->56 + + + + + +57 + +plot_split_images +apa: 2 +cmap: coolwarm +event: 0 +ext: png +plane: U + + + +38->57 + + + + + +58 + +plot_split_images +apa: 2 +cmap: viridis +event: 0 +ext: png +plane: U + + + +38->58 + + + + + +59 + +plot_split_images +apa: 2 +cmap: seismic +event: 0 +ext: pdf +plane: U + + + +38->59 + + + + + +60 + +plot_split_images +apa: 2 +cmap: Spectral +event: 0 +ext: pdf +plane: U + + + +38->60 + + + + + +61 + +plot_split_images +apa: 2 +cmap: terrain +event: 0 +ext: pdf +plane: U + + + +38->61 + + + + + +62 + +plot_split_images +apa: 2 +cmap: coolwarm +event: 0 +ext: pdf +plane: U + + + +38->62 + + + + + +63 + +plot_split_images +apa: 2 +cmap: viridis +event: 0 +ext: pdf +plane: U + + + +38->63 + + + + + +64 + +plot_split_images +apa: 2 +cmap: seismic +event: 0 +ext: svg +plane: U + + + +38->64 + + + + + +65 + +plot_split_images +apa: 2 +cmap: Spectral +event: 0 +ext: svg +plane: U + + + +38->65 + + + + + +66 + +plot_split_images +apa: 2 +cmap: terrain +event: 0 +ext: svg +plane: U + + + +38->66 + + + + + +67 + +plot_split_images +apa: 2 +cmap: coolwarm +event: 0 +ext: svg +plane: U + + + +38->67 + + + + + +68 + +plot_split_images +apa: 2 +cmap: viridis +event: 0 +ext: svg +plane: U + + + +38->68 + + - + + +38->69 + + + + -30->9 - - +39->0 + + - + -31->9 - - +40->0 + + - + -32->9 - - - - - -39 - -plot_frames_hidpi - - - -32->39 - - +41->0 + + - + -33->9 - - +42->0 + + - + -34->9 - - +43->0 + + - + -35->9 - - +44->0 + + - + -36->9 - - +45->0 + + - + -37->9 - - +46->0 + + - + -38->9 - - +47->0 + + - + -39->9 - - +48->0 + + + + + +49->0 + + + + + +50->0 + + - + -40->9 - - +51->0 + + - + -41->9 - - +52->0 + + - + -42->9 - - +53->0 + + - + -43->9 - - +54->0 + + - + -44->9 - - +55->0 + + - + -45->9 - - +56->0 + + - + -46->9 - - +57->0 + + - + -47->9 - - +58->0 + + - + -48->9 - - +59->0 + + - + -49->9 - - +60->0 + + - + -50->9 - - +61->0 + + - + -51->9 - - +62->0 + + - + -52->9 - - +63->0 + + - + -53->9 - - +64->0 + + - + -54->9 - - +65->0 + + - + -55->9 - - +66->0 + + - + -56->9 - - +67->0 + + - + -57->9 - - - - - -58->9 - - - - - -59->9 - - - - - -60->9 - - - - - -61->9 - - - - - -62->9 - - - - - -63->9 - - - - - -64->9 - - - - - -65->9 - - - - - -66->9 - - - - - -67->9 - - - - - -68->9 - - - - - -69->9 - - +68->0 + + diff --git a/toyzero.yaml b/toyzero.yaml index 761206a..6da7d11 100644 --- a/toyzero.yaml +++ b/toyzero.yaml @@ -3,11 +3,14 @@ # This is the default toyzero config file. # Use like: # snakemake --configfile mycfg.yaml -jall all +# Or further, like +# snakemake --configfile mycfg.yaml -jall all --config outdir=other +# See top of snakefile for what can be set here. outdir: "." -datadir: "data" -plotdir: "plots" +# best to set something unique on the CLI!!! +# seed will be used as an intermediate under outdir seed: "1,2,3,4" ntracks: 10 -nevents: 10 -# how noisy wire-cell should be. "debug" is more, "error" is less -wcloglvl: info +nevents: 1 +# how noisy wire-cell should be. "debug" is more, "info" is less +wcloglvl: debug