diff --git a/.gitignore b/.gitignore index 82f1c51..1aaf6f2 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,7 @@ O.*/ *~ *.swp +*.pyc *BAK.adl cdCommands envPaths diff --git a/configure/RULES_BUILD b/configure/RULES_BUILD index eba3adc..a3c3636 100644 --- a/configure/RULES_BUILD +++ b/configure/RULES_BUILD @@ -8,6 +8,11 @@ ifdef IOCADMIN INSTALL_IOCRELEASE_DBS=$(IOCRELEASE_DB:%=$(INSTALL_DB)/%) +PYTHON=$(shell which python2) +ifneq ("python2","$(notdir $(PYTHON))") + PYTHON=python +endif + #-------------------------------------------------- # Rules @@ -15,9 +20,9 @@ build : $(IOCRELEASE_DB) buildInstall : $(INSTALL_IOCRELEASE_DBS) # Create database for module version PVs -$(IOCRELEASE_DB) : $(TOP)/configure/RELEASE $(TOP)/RELEASE_SITE +$(IOCRELEASE_DB) : $(TOP)/configure/RELEASE $(RM) $@ - $(IOCADMIN)/bin/$(EPICS_HOST_ARCH)/iocReleaseCreateDb.py $^ >$@ + $(PYTHON) $(IOCADMIN)/bin/$(EPICS_HOST_ARCH)/iocReleaseCreateDb.py $^ > $@ $(INSTALL_DB)/%: % @echo "Installing db file $@" diff --git a/iocAdmin/src/Makefile b/iocAdmin/src/Makefile index 89859ee..0b8a1b7 100644 --- a/iocAdmin/src/Makefile +++ b/iocAdmin/src/Makefile @@ -14,6 +14,8 @@ DBD += iocAdmin.dbd #============================= SCRIPTS_HOST += iocReleaseCreateDb.py +SCRIPTS_HOST += pkgNamesToMacroNames.py +SCRIPTS_HOST += version_utils.py #LIBRARY = iocAdmin diff --git a/iocAdmin/src/iocReleaseCreateDb.py b/iocAdmin/src/iocReleaseCreateDb.py old mode 100644 new mode 100755 index f746617..4fe750d --- a/iocAdmin/src/iocReleaseCreateDb.py +++ b/iocAdmin/src/iocReleaseCreateDb.py @@ -1,25 +1,26 @@ -#!/usr/bin/env python +#!/usr/bin/python2 import sys import os import subprocess import optparse +from pkgNamesToMacroNames import * +from version_utils import * +__all__ = ['export_db_file', 'process_options'] -__all__ = ['export_db_file', 'module_versions', 'process_options'] - -def export_db_file(module_versions, path=None): +def export_db_file( module_versions, path=None): """ Use the contents of a dictionary of module versions to create a database - of module release stringin PVs. The database + of module release stringin PVs. The database is written to stdout if path is not provided or is None. """ out_file = sys.stdout idx = 0 - idxMax = 20 - + idxMax = 30 + if path: try: out_file = open(path, 'w') @@ -27,7 +28,8 @@ def export_db_file(module_versions, path=None): sys.stderr.write('Could not open "%s": %s\n' % (path, e.strerror)) return None - sorted_module_versions = [(key, module_versions[key]) for key in sorted(module_versions.keys())] + # Create a sorted list of versions + sorted_module_versions = [(key, module_versions[key]) for key in sorted(module_versions.keys())] print >> out_file, '#==============================================================================' print >> out_file, '#' @@ -43,7 +45,8 @@ def export_db_file(module_versions, path=None): strip off the _MODULE_VERSION from key for PV NAME """ x = key.replace("_MODULE_VERSION","",1) - if idx >= idxMax: break + if idx >= idxMax: + break print >> out_file, 'record(stringin, "$(IOC):RELEASE%02d") {' % idx print >> out_file, ' field(DESC, "%s")' % x print >> out_file, ' field(PINI, "YES")' @@ -51,7 +54,7 @@ def export_db_file(module_versions, path=None): print >> out_file, ' #field(ASG, "some read only grp")' print >> out_file, '}' idx = idx + 1 - + while idx < idxMax: print >> out_file, 'record(stringin, "$(IOC):RELEASE%02d") {' % idx print >> out_file, ' field(DESC, "Not Applicable")' @@ -60,79 +63,9 @@ def export_db_file(module_versions, path=None): print >> out_file, ' #field(ASG, "some read only grp")' print >> out_file, '}' idx = idx + 1 - + if out_file != sys.stdout: out_file.close() - - -def module_versions(release_path, site_path): - """ - Return a dictionary containing module names and versions. - """ - - # first grab EPICS_BASE_VER from RELEASE_SITE file, if it's there - siteBaseVer = "Nada" - openSiteFile = 1 - - try: - site_file = open(site_path, 'r') - except IOError, e: - #sys.stderr.write('Could not open "%s": %s\n' % (site_path, e.strerror)) - openSiteFile = 0 - - if openSiteFile: - for line in site_file: - # Remove comments - line = line.partition('#')[0] - - # Turn 'a = b' into a key/value pair and remove leading and trailing whitespace - (key, sep, value) = line.partition('=') - key = key.strip() - value = value.strip() - - # save EPICS_BASE_VER, if it's in there - if key.startswith('EPICS_BASE_VER'): - siteBaseVer = value - break - - site_file.close() - - # now get all the modules - try: - release_file = open(release_path, 'r') - except IOError, e: - sys.stderr.write('Could not open "%s": %s\n' % (release_path, e.strerror)) - return None - - release_file_dict = {} - - for line in release_file: - # Remove comments - line = line.partition('#')[0] - - # Turn 'a = b' into a key/value pair and remove leading and trailing whitespace - (key, sep, value) = line.partition('=') - key = key.strip() - value = value.strip() - - # Add the key/value pair to the dictionary if the key ends with _MODULE_VERSION - if key.endswith('_MODULE_VERSION'): - # if BASE_MODULE_VERSION is set to EPICS_BASE_VER macro from RELEASE_SITE, - # capture it here - if key == "BASE_MODULE_VERSION" and value == "$(EPICS_BASE_VER)": - if siteBaseVer != "Nada": - release_file_dict[key] = siteBaseVer - else: - # don't set BASE at all - pass - else: - release_file_dict[key] = value - - release_file.close() - - - return release_file_dict - def process_options(argv): """ @@ -143,9 +76,8 @@ def process_options(argv): if argv is None: argv = sys.argv[1:] - # usage = 'Usage: %prog RELEASE_FILE [options]' - usage = 'Usage: %prog RELEASE_FILE RELEASE_SITE_FILE [options]' - version = '%prog 0.1' + usage = 'Usage: %prog RELEASE_FILE [options]' + version = '%prog 0.2' parser = optparse.OptionParser(usage=usage, version=version) parser.add_option('-v', '--verbose', action='store_true', dest='verbose', help='print verbose output') @@ -156,22 +88,25 @@ def process_options(argv): (options, args) = parser.parse_args(argv) - if len(args) != 2: + if len(args) != 1: parser.error("incorrect number of arguments") options.release_file_path = os.path.normcase(args[0]) - options.release_site_file_path = os.path.normcase(args[1]) - - return options + return options def main(argv=None): options = process_options(argv) - versions = module_versions(options.release_file_path, options.release_site_file_path) - export_db_file(versions, options.db_file) + + # get the IOC dependents + topDir = os.path.abspath( os.path.dirname( os.path.dirname(options.release_file_path) ) ) + dependents = getEpicsPkgDependents( topDir ) + + # export the iocRelease.db file + export_db_file( dependents, options.db_file) return 0 - + if __name__ == '__main__': status = main() diff --git a/iocAdmin/src/pkgNamesToMacroNames.py b/iocAdmin/src/pkgNamesToMacroNames.py new file mode 100644 index 0000000..324b6b2 --- /dev/null +++ b/iocAdmin/src/pkgNamesToMacroNames.py @@ -0,0 +1,288 @@ +'''This file facilitates translating EPICS macro names from RELEASE files, +typically EPICS modules, to the name of the package it's used for. +Additional macro names can be added to this list as desired. +Packages can have multiple macro names which all get mapped to that package name. +Macro names not found are converted to lower-case and returned as the package name. +''' +import os +import string + +_macroNameToPkgName = {} +_pkgNameToMacroNames = {} + +# These macroNames are never mapped to pkgNames +_macroNameToPkgName[ 'ALARM_CONFIGS_TOP' ] = None +_macroNameToPkgName[ 'BASE_SITE_TOP' ] = None +_macroNameToPkgName[ 'CONFIG' ] = None +_macroNameToPkgName[ 'CONFIG_SITE_TOP' ] = None +_macroNameToPkgName[ 'EPICS_BASE_VER' ] = None +_macroNameToPkgName[ 'EPICS_EXTENSIONS' ] = None +_macroNameToPkgName[ 'EPICS_MODULES' ] = None +_macroNameToPkgName[ 'EPICS_SITE_TOP' ] = None +_macroNameToPkgName[ 'EVR_MODULE' ] = None +_macroNameToPkgName[ 'IOC_SITE_TOP' ] = None +_macroNameToPkgName[ 'LINUX_KERNEL_MODULES' ] = None +_macroNameToPkgName[ 'MAKE_TEST_IOC_APP' ] = None +_macroNameToPkgName[ 'MY_MODULES' ] = None +_macroNameToPkgName[ 'PACKAGE_SITE_TOP' ] = None +_macroNameToPkgName[ 'PSPKG_ROOT' ] = None +_macroNameToPkgName[ 'RULES' ] = None +_macroNameToPkgName[ 'TEMPLATE_TOP' ] = None +_macroNameToPkgName[ 'TOOLS_SITE_TOP' ] = None +_macroNameToPkgName[ 'TOP' ] = None + +def macroNameToPkgName( macroName ): + if macroName in _macroNameToPkgName: + pkgName = _macroNameToPkgName[ macroName ] + else: + pkgName = macroName.lower() + return pkgName + +def pkgNameGetMacroNames( pkgName ): + if pkgName in _pkgNameToMacroNames: + macroNames = _pkgNameToMacroNames[ pkgName ] + if not pkgName.upper() in macroNames: + macroNames.append( pkgName.upper() ) + else: + macroNames = [ pkgName.upper() ] + return macroNames + +def pkgNameAddMacroName( pkgName, macroName ): + if not pkgName in _pkgNameToMacroNames: + _pkgNameToMacroNames[pkgName] = [ macroName ] + else: + macroNames = _pkgNameToMacroNames[pkgName] + if not macroName in macroNames: + macroNames.append( macroName ) + _pkgNameToMacroNames[pkgName] = macroNames + + if not macroName in _macroNameToPkgName: + _macroNameToPkgName[macroName] = pkgName + else: + if _macroNameToPkgName[macroName] != pkgName: + if _macroNameToPkgName[macroName] is None: + print("pkgNameAddMacroName Error: Pkg %s Macro %s is not a valid pkgName" % \ + ( pkgName, macroName )) + else: + print("pkgNameAddMacroName Error: Pkg %s Macro %s already mapped to %s" % \ + ( pkgName, macroName, _macroNameToPkgName[macroName] )) + +# Populate macro names for packages +# Most of these are all just simple mappings to uppercase +pkgNameAddMacroName( 'acqiris', 'ACQIRIS' ) +pkgNameAddMacroName( 'ADCore', 'ADCORE' ) +pkgNameAddMacroName( 'ADCrayDl', 'ADCRAYDL' ) +pkgNameAddMacroName( 'ADCSimDetector', 'ADCSIMDETECTOR' ) +pkgNameAddMacroName( 'ADEdtPdv', 'ADEDTPDV' ) +pkgNameAddMacroName( 'ADPgpEdt', 'ADPGPEDT' ) +pkgNameAddMacroName( 'ADPointGrey', 'ADPOINTGREY' ) +pkgNameAddMacroName( 'ADProsilica', 'ADPROSILICA' ) +pkgNameAddMacroName( 'ADS', 'ADS' ) +pkgNameAddMacroName( 'ADSimDetector', 'ADSIMDETECTOR' ) +pkgNameAddMacroName( 'ADStream', 'ADSTREAM' ) +pkgNameAddMacroName( 'ADSupport', 'ADSUPPORT' ) +pkgNameAddMacroName( 'ADUtil', 'ADUTIL' ) +pkgNameAddMacroName( 'ADVisar', 'ADVISAR' ) +pkgNameAddMacroName( 'agilent53210a', 'AGILENT53210A' ) +pkgNameAddMacroName( 'agilent53220A', 'AGILENT53220A' ) +pkgNameAddMacroName( 'anc350', 'ANC350' ) +pkgNameAddMacroName( 'apci1710', 'APCI1710' ) +pkgNameAddMacroName( 'aravisGigE', 'ARAVISGIGE' ) +pkgNameAddMacroName( 'areaDetectorSupp', 'AREADETECTORSUPP' ) +pkgNameAddMacroName( 'asynDribble', 'ASYNDRIBBLE' ) +pkgNameAddMacroName( 'asynGenicam', 'ASYNGENICAM' ) +pkgNameAddMacroName( 'asyn', 'ASYN' ) +pkgNameAddMacroName( 'autosave', 'AUTOSAVE' ) +pkgNameAddMacroName( 'average', 'AVERAGE' ) +pkgNameAddMacroName( 'axis', 'AXIS' ) +pkgNameAddMacroName( 'bacnet', 'BACNET' ) +pkgNameAddMacroName( 'BergozBCM-RF-asyn', 'BERGOZBCM-RF-ASYN' ) +pkgNameAddMacroName( 'bkhAsyn', 'BKHASYN' ) +pkgNameAddMacroName( 'bldClient', 'BLDCLIENT' ) +pkgNameAddMacroName( 'boost-wrapper', 'BOOST-WRAPPER' ) +pkgNameAddMacroName( 'bpmDigitizer', 'BPMDIGITIZER' ) +pkgNameAddMacroName( 'BsaCore', 'BSACORE' ) +pkgNameAddMacroName( 'bsaDriver', 'BSADRIVER' ) +pkgNameAddMacroName( 'busy', 'BUSY' ) +pkgNameAddMacroName( 'Bx9000_MBT', 'BX9000_MBT' ) +pkgNameAddMacroName( 'caenADCV965', 'CAENADCV965' ) +pkgNameAddMacroName( 'calc', 'CALC' ) +pkgNameAddMacroName( 'caPutLog', 'CAPUTLOG' ) +pkgNameAddMacroName( 'cexpsh', 'CEXPSH' ) +pkgNameAddMacroName( 'cpswAsynPortDriver', 'CPSWASYNPORTDRIVER' ) +pkgNameAddMacroName( 'cpsw-wrapper', 'CPSW-WRAPPER' ) +pkgNameAddMacroName( 'crossbarControl', 'CROSSBARCONTROL' ) +pkgNameAddMacroName( 'devBusMapped', 'DEVBUSMAPPED' ) +pkgNameAddMacroName( 'devGenVar', 'DEVGENVAR' ) +pkgNameAddMacroName( 'devlib2', 'DEVLIB2' ) +pkgNameAddMacroName( 'diagTimer', 'DIAGTIMER' ) +pkgNameAddMacroName( 'drvPciMcor', 'DRVPCIMCOR' ) +pkgNameAddMacroName( 'drvUioPciGen', 'DRVUIOPCIGEN' ) +pkgNameAddMacroName( 'EDT_CL', 'EDT_CL' ) +pkgNameAddMacroName( 'epm2000', 'EPM2000' ) +pkgNameAddMacroName( 'ethercat', 'ETHERCAT' ) +pkgNameAddMacroName( 'ether_ip', 'ETHER_IP' ) +pkgNameAddMacroName( 'etherPSC', 'ETHERPSC' ) +pkgNameAddMacroName( 'ev2_driver', 'EV2_DRIVER' ) +pkgNameAddMacroName( 'event2', 'EVENT2' ) +pkgNameAddMacroName( 'event', 'EVENT' ) +pkgNameAddMacroName( 'evrClient', 'EVRCLIENT' ) +pkgNameAddMacroName( 'exampleCPP', 'EXAMPLECPP' ) +pkgNameAddMacroName( 'fcom', 'FCOM' ) +pkgNameAddMacroName( 'fcomUtil', 'FCOMUTIL' ) +pkgNameAddMacroName( 'ffmpegServer', 'FFMPEGSERVER' ) +pkgNameAddMacroName( 'genPolySub', 'GENPOLYSUB' ) +pkgNameAddMacroName( 'genSub', 'GENSUB' ) +pkgNameAddMacroName( 'gtr', 'GTR' ) +pkgNameAddMacroName( 'highlandLVDTV550', 'HIGHLANDLVDTV550' ) +pkgNameAddMacroName( 'history', 'HISTORY' ) +pkgNameAddMacroName( 'Hp53181A', 'HP53181A' ) +pkgNameAddMacroName( 'hpsTpr-wrapper', 'HPSTPR-WRAPPER' ) +pkgNameAddMacroName( 'hytec8413', 'HYTEC8413' ) +pkgNameAddMacroName( 'icdTemplates', 'ICDTEMPLATES' ) +pkgNameAddMacroName( 'InternalData', 'INTERNALDATA' ) +pkgNameAddMacroName( 'iocAdmin', 'IOCADMIN' ) +pkgNameAddMacroName( 'iocStats', 'IOCSTATS' ) +pkgNameAddMacroName( 'ip231', 'IP231' ) +pkgNameAddMacroName( 'ip330-asyn', 'IP330-ASYN' ) +pkgNameAddMacroName( 'ip330', 'IP330' ) +pkgNameAddMacroName( 'ip440-asyn', 'IP440-ASYN' ) +pkgNameAddMacroName( 'ip440', 'IP440' ) +pkgNameAddMacroName( 'ip445-asyn', 'IP445-ASYN' ) +pkgNameAddMacroName( 'ip445', 'IP445' ) +pkgNameAddMacroName( 'ip470', 'IP470' ) +pkgNameAddMacroName( 'ipac', 'IPAC' ) +pkgNameAddMacroName( 'ipimb', 'IPIMB' ) +pkgNameAddMacroName( 'ipmiComm', 'IPMICOMM' ) +pkgNameAddMacroName( 'JitterCleaner', 'JITTERCLEANER' ) +pkgNameAddMacroName( 'Keithley6482', 'KEITHLEY6482' ) +pkgNameAddMacroName( 'Keithley6487', 'KEITHLEY6487' ) +pkgNameAddMacroName( 'l2MpsAsyn', 'L2MPSASYN' ) +pkgNameAddMacroName( 'laserLocking', 'LASERLOCKING' ) +pkgNameAddMacroName( 'lcls2-timing-bsa-wrapper', 'LCLS2-TIMING-BSA-WRAPPER' ) +pkgNameAddMacroName( 'LeCroy_ENET', 'LECROY_ENET' ) +pkgNameAddMacroName( 'LLRFLibs', 'LLRFLIBS' ) +pkgNameAddMacroName( 'MCoreUtils', 'MCOREUTILS' ) +pkgNameAddMacroName( 'miscUtils', 'MISCUTILS' ) +pkgNameAddMacroName( 'mksu', 'MKSU' ) +pkgNameAddMacroName( 'MMC', 'MMC' ) +pkgNameAddMacroName( 'modbus', 'MODBUS' ) +pkgNameAddMacroName( 'ModBusTCPClnt', 'MODBUSTCPCLNT' ) +pkgNameAddMacroName( 'motor', 'MOTOR' ) +pkgNameAddMacroName( 'mrfioc2', 'MRFIOC2' ) +pkgNameAddMacroName( 'NDDriverStdArrays', 'NDDRIVERSTDARRAYS' ) +pkgNameAddMacroName( 'normativeTypesCPP', 'NORMATIVETYPESCPP' ) +pkgNameAddMacroName( 'nullhttpd', 'NULLHTTPD' ) +pkgNameAddMacroName( 'oxigrafMO2i', 'OXIGRAFMO2I' ) +pkgNameAddMacroName( 'p4p', 'P4P' ) +pkgNameAddMacroName( 'pau', 'PAU' ) +pkgNameAddMacroName( 'pcds_motion', 'PCDS_MOTION' ) +pkgNameAddMacroName( 'perfMeasure', 'PERFMEASURE' ) +pkgNameAddMacroName( 'pgp', 'PGP' ) +pkgNameAddMacroName( 'plcAdmin', 'PLCADMIN' ) +pkgNameAddMacroName( 'pva2pva', 'PVA2PVA' ) +pkgNameAddMacroName( 'pvAccessCPP', 'PVACCESSCPP' ) +pkgNameAddMacroName( 'pvaClientCPP', 'PVACLIENTCPP' ) +pkgNameAddMacroName( 'pvaClientTestCPP', 'PVACLIENTTESTCPP' ) +pkgNameAddMacroName( 'pvaPy', 'PVAPY' ) +pkgNameAddMacroName( 'pvaSrv', 'PVASRV' ) +pkgNameAddMacroName( 'pvCommonCPP', 'PVCOMMONCPP' ) +pkgNameAddMacroName( 'pvDatabaseCPP', 'PVDATABASECPP' ) +pkgNameAddMacroName( 'pvDataCPP', 'PVDATACPP' ) +pkgNameAddMacroName( 'pvExampleTestCPP', 'PVEXAMPLETESTCPP' ) +pkgNameAddMacroName( 'QF2preMonitor', 'QF2PREMONITOR' ) +pkgNameAddMacroName( 'quadEM', 'QUADEM' ) +pkgNameAddMacroName( 'RFControlFirmware', 'RFCONTROLFIRMWARE' ) +pkgNameAddMacroName( 'RFControl', 'RFCONTROL' ) +pkgNameAddMacroName( 'rtemsutils', 'RTEMSUTILS' ) +pkgNameAddMacroName( 's7plc', 'S7PLC' ) +pkgNameAddMacroName( 'seq', 'SEQ' ) +pkgNameAddMacroName( 'sis8300', 'SIS8300' ) +pkgNameAddMacroName( 'snmp', 'SNMP' ) +pkgNameAddMacroName( 'spline', 'SPLINE' ) +pkgNameAddMacroName( 'sr620', 'SR620' ) +pkgNameAddMacroName( 'sscan', 'SSCAN' ) +pkgNameAddMacroName( 'sSubRecord', 'SSUBRECORD' ) +pkgNameAddMacroName( 'std', 'STD' ) +pkgNameAddMacroName( 'streamdevice', 'STREAMDEVICE' ) +pkgNameAddMacroName( 'synapps-support', 'SYNAPPS-SUPPORT' ) +pkgNameAddMacroName( 'tds3000', 'TDS3000' ) +pkgNameAddMacroName( 'teste', 'TESTE' ) +pkgNameAddMacroName( 'timeStampFifo', 'TIMESTAMPFIFO' ) +pkgNameAddMacroName( 'timesync', 'TIMESYNC' ) +pkgNameAddMacroName( 'timingApi', 'TIMINGAPI' ) +pkgNameAddMacroName( 'tpr', 'TPR' ) +pkgNameAddMacroName( 'tprPattern', 'TPRPATTERN' ) +pkgNameAddMacroName( 'tprTrigger', 'TPRTRIGGER' ) +pkgNameAddMacroName( 'TRCore', 'TRCORE' ) +pkgNameAddMacroName( 'TRGeneralStandards', 'TRGENERALSTANDARDS' ) +pkgNameAddMacroName( 'twincat-ads', 'TWINCAT-ADS' ) +pkgNameAddMacroName( 'udpComm', 'UDPCOMM' ) +pkgNameAddMacroName( 'usb_sn', 'USB_SN' ) +pkgNameAddMacroName( 'vacuum', 'VACUUM' ) +pkgNameAddMacroName( 'VHSx0x', 'VHSX0X' ) +pkgNameAddMacroName( 'vmeCardRecord', 'VMECARDRECORD' ) +pkgNameAddMacroName( 'vsam', 'VSAM' ) +pkgNameAddMacroName( 'xipIo', 'XIPIO' ) +pkgNameAddMacroName( 'xps8', 'XPS8' ) +pkgNameAddMacroName( 'yaml-cpp-wrapper', 'YAML-CPP-WRAPPER' ) +pkgNameAddMacroName( 'yamlDownloader', 'YAMLDOWNLOADER' ) +pkgNameAddMacroName( 'yamlLoader', 'YAMLLOADER' ) +pkgNameAddMacroName( 'ycpswasyn', 'YCPSWASYN' ) + +# Add special cases +pkgNameAddMacroName( 'areaDetector', 'AREA_DETECTOR' ) +pkgNameAddMacroName( 'base', 'BASE' ) +pkgNameAddMacroName( 'base', 'BASE_MODULE_VERSION' ) +pkgNameAddMacroName( 'base', 'EPICS_BASE' ) +pkgNameAddMacroName( 'BergozBCM-RF-asyn','BERGOZBCM_RF_ASYN' ) +pkgNameAddMacroName( 'Bk9000_MBT', 'BK9000_MBT' ) +pkgNameAddMacroName( 'bldClient', 'BLD_CLIENT' ) +pkgNameAddMacroName( 'Bx9000_MBT', 'BX9000' ) +pkgNameAddMacroName( 'Bx9000_MBT', 'BX9000_MBT' ) +pkgNameAddMacroName( 'Bx9000_MBT', 'BX9000MBT' ) +pkgNameAddMacroName( 'Camcom', 'CAMCOM' ) +pkgNameAddMacroName( 'caSnooper', 'CASNOOPER' ) +pkgNameAddMacroName( 'cexpsh', 'CEXP' ) +pkgNameAddMacroName( 'ChannelWatcher', 'CHANNELWATCHER' ) +pkgNameAddMacroName( 'diagTimer', 'DIAG_TIMER' ) +pkgNameAddMacroName( 'etherPSC', 'EPSC' ) +pkgNameAddMacroName( 'ethercat', 'ECASYN' ) +pkgNameAddMacroName( 'fwdCliS', 'FWDCLIS' ) +pkgNameAddMacroName( 'gtr', 'GTR_VERSION' ) +pkgNameAddMacroName( 'ip231-asyn', 'IP231_ASYN' ) +pkgNameAddMacroName( 'ip330-asyn', 'IP330_ASYN' ) +pkgNameAddMacroName( 'ip440', 'XY2440' ) +pkgNameAddMacroName( 'ip440-asyn', 'IP440_ASYN' ) +pkgNameAddMacroName( 'ip445', 'XY2445' ) +pkgNameAddMacroName( 'ip445-asyn', 'IP445_ASYN' ) +pkgNameAddMacroName( 'LeCroy_ENET', 'LECROY' ) +pkgNameAddMacroName( 'normativeTypesCPP','NORMATIVE' ) +pkgNameAddMacroName( 'normativeTypesCPP','NORMATIVETYPES' ) +pkgNameAddMacroName( 'procServ', 'PROCSERV' ) +pkgNameAddMacroName( 'PSCD_Camac', 'PSCDCAMAC' ) +pkgNameAddMacroName( 'pvAccessCPP', 'PVACCESS' ) +pkgNameAddMacroName( 'pvAccessCPP', 'pvAccessCPP' ) +pkgNameAddMacroName( 'pvaClientCPP', 'PVACLIENT' ) +pkgNameAddMacroName( 'pvaSrv', 'PVASRV' ) +pkgNameAddMacroName( 'pvCommonCPP', 'PVCOMMONCPP' ) +pkgNameAddMacroName( 'pvCommonCPP', 'PVCOMMON' ) +pkgNameAddMacroName( 'pvCommonCPP', 'pvCommonCPP' ) +pkgNameAddMacroName( 'pvDatabaseCPP', 'PVDATABASECPP' ) +pkgNameAddMacroName( 'pvDatabaseCPP', 'PVDATABASE' ) +pkgNameAddMacroName( 'pvDataCPP', 'PVDATACPP' ) +pkgNameAddMacroName( 'pvDataCPP', 'PVDATA' ) +pkgNameAddMacroName( 'pvDataCPP', 'pvDataCPP' ) +pkgNameAddMacroName( 'pvIOCCPP', 'PVIOC' ) +pkgNameAddMacroName( 'seq', 'SNCSEQ' ) +pkgNameAddMacroName( 'sscan', 'SSCAN' ) +pkgNameAddMacroName( 'sSubRecord', 'SSUBRECORD' ) +pkgNameAddMacroName( 'streamdevice', 'STREAMDEVICE' ) +pkgNameAddMacroName( 'streamdevice', 'STREAM' ) +pkgNameAddMacroName( 'StripTool', 'STRIPTOOL' ) +pkgNameAddMacroName( 'timingApi', 'TIMING_API' ) +pkgNameAddMacroName( 'VHQx0x', 'VHQX0X' ) +pkgNameAddMacroName( 'VHSx0x', 'VHSX0X' ) +pkgNameAddMacroName( 'VisualDCT', 'VISUALDCT' ) +pkgNameAddMacroName( 'vmeCardRecord', 'VME_CARD_RECORD' ) + diff --git a/iocAdmin/src/version_utils.py b/iocAdmin/src/version_utils.py new file mode 100644 index 0000000..b9d0557 --- /dev/null +++ b/iocAdmin/src/version_utils.py @@ -0,0 +1,371 @@ +#!/usr/bin/env python +#from __future__ import print_function +import os +import re +import sys +import glob +import subprocess +from pkgNamesToMacroNames import * +# +# Purpose: +# +# Utilities for analyzing version and release tags +# +# Copyright 2016,2017,2018 Stanford University +# Author: Bruce Hill +# +# Released under the GPLv2 licence +# + + +# Pre-compile regular expressions for speed +numberRegExp = re.compile( r"(\d+)" ) +releaseRegExp = re.compile( r"(|[a-zA-Z0-9_-]*-)R(\d+)[-_.](\d+)(.*)" ) +macroNameRegExp = re.compile( r"^\s*([a-zA-Z0-9_]*)\s*=\s*(\S*)\s*$" ) +condMacroRegExp = re.compile( r"^(#*)\s*([a-zA-Z0-9_]+)\s*=\s*(\S*)\s*$" ) +macroRefRegExp = re.compile( r"^(.*)\$\(([a-zA-Z0-9_]+)\)(.*)$" ) +moduleVersionRegExp = re.compile( r"^\s*([a-zA-Z0-9_]+)_MODULE_VERSION\s*=\s*(\S*)\s*$" ) +epicsBaseVerRegExp = re.compile( r"^\s*([A-Za-z0-9_-]*BASE[A-Za-z0-9_-]*VER[SION]*)\s*=\s*(\S*)\s*$" ) +epicsModulesRegExp = re.compile( r"^\s*EPICS_MODULES\s*=\s*(\S*\s*)$" ) +modulesSiteTopRegExp= re.compile( r"^\s*MODULES_SITE_TOP\s*=\s*(\S*\s*)$" ) +versionRegExp = re.compile( r"^\s*([A-Za-z0-9_-]*VERSION)\s*=\s*(\S*)\s*$" ) + +def VersionToRelNumber( version, debug=False ): + relNumber = 0.0 + try: + ver = version + if debug: + print("VersionToRelNumber: %s" % ( ver )) + verMatch = releaseRegExp.search( ver ) + if verMatch: + ver = verMatch.group(2) + '.' + verMatch.group(3) + verMatch.group(4) + ver = ver.replace( '-', '.' ) + ver = ver.replace( '_', '.' ) + verNumbers = ver.split( '.' ) + scale = 1.0 + for n in verNumbers: + m = numberRegExp.search( n ) + if m and m.group(1): + relNumber += float(m.group(1)) / scale + scale *= 100.0 + except: + pass + if debug: + print("VersionToRelNumber: %s = %f" % ( version, relNumber )) + return relNumber + +def isReleaseCandidate(release): + if release.endswith( "FAILED" ): + return False + match = releaseRegExp.search( release ) + if match: + return True + +def isBaseTop(path): + '''isBaseTop does a simple check for startup/EpicsHostArch. + More tests can be added if needed.''' + if os.path.isfile( os.path.join( path, 'startup', 'EpicsHostArch' ) ): + return True + return False + +def isEpicsPackage(path): + '''isEpicsPackage does a simple check for configure/RELEASE. + More tests can be added if needed.''' + if os.path.isfile( os.path.join( path, 'configure', 'RELEASE' ) ): + return True + return False + +def get_base_versions( epics_site_top ): + base_versions = [] + base_candidates = os.listdir( os.path.join( epics_site_top, 'base' ) ) + for base_candidate in base_candidates: + if isBaseTop( os.path.join( epics_site_top, 'base', base_candidate ) ): + base_versions.append( base_candidate ) + return base_versions + +def determine_epics_base_ver(): + '''Returns EPICS base version string, or None if unable to derive.''' + # If we have EPICS_BASE, work back from there + epics_base = os.getenv('EPICS_BASE') + if epics_base: + epics_base_ver = os.path.basename( os.path.normpath(epics_base) ) + return epics_base_ver + + # If not, look for EPICS_BASE_VER in the environment + epics_base_ver = os.getenv('EPICS_BASE_VER') + # Then EPICS_VER + if not epics_base_ver: + epics_base_ver = os.getenv('EPICS_VER') + # Then BASE_MODULE_VERSION + if not epics_base_ver: + epics_base_ver = os.getenv('BASE_MODULE_VERSION') + + # Returns None if not found + return epics_base_ver + +def expandMacros( strWithMacros, macroDict ): + while True: + macroMatch = macroRefRegExp.search( strWithMacros ) + if not macroMatch: + break + macroName = macroMatch.group(2) + if not macroName in macroDict: + # No need to expand other macros in the string once one has failed + break + # Expand this macro and continue + strWithMacros = macroMatch.group(1) + macroDict[macroName] + macroMatch.group(3) + + return strWithMacros + +def getPkgReleaseList( top, pkgName ): + '''For a given top directory, add pkgName to top and look + for EPICS releases in that directory. + Returns a sorted list of releases, most recent first.''' + # Loop through the directories looking for releases + if not os.path.isdir( top ): + print("getPkgReleaseList Error: top is not a directory: %s\n" % top) + pkgDir = os.path.join( top, pkgName ) + if not os.path.isdir( pkgDir ): + print("getPkgReleaseList Error: %s is not a package under %s\n" % ( pkgName, top )) + + releaseList = [ ] + for dirPath, dirs, files in os.walk( pkgDir, topdown=True ): + if len( dirs ) == 0: + continue + if '.git' in dirs: + dirs.remove( '.git' ) + if '.svn' in dirs: + dirs.remove( '.svn' ) + if 'CVS' in dirs: + dirs.remove( 'CVS' ) + releases = [ ] + dirs.sort() + for dir in dirs[:]: + # Remove from list so we don't search recursively + dirs.remove( dir ) + if not isReleaseCandidate(dir): + continue + release = os.path.join( dirPath, dir ) + verPath = os.path.join( release, "configure", "RELEASE" ) + + buildPath = os.path.join( release, "build" ) + if os.path.isfile( verPath ) or os.path.isdir( buildPath ): + releases += [ release ] + + if len( releases ) == 0: + continue; + + # Create the release set so we can order the releases by version number + releaseSet = { } + for release in releases: + ( reldir, ver ) = os.path.split( release ) + relNumber = VersionToRelNumber( ver ) + while relNumber in releaseSet: + relNumber -= 1e-12 + releaseSet[ relNumber ] = release + + for release in sorted( list(releaseSet.keys()), reverse = True ): + releaseList += [ releaseSet[ release ] ] + return releaseList + +def getMacrosFromFile( filePath, macroDict, debug = False, required = False ): + '''Find and return a dictionary of gnu make style macros + found in a file. Ex. macroDict['BASE_MODULE_VERSION'] = 'R3.15.5-1.0' + ''' + if not os.path.isfile( filePath ): + if required: + print("getMacrosFromFile Error: unable to open %s" % filePath) + return macroDict + if debug: + print("getMacrosFromFile %s: %d versions on entry" % ( filePath, len(macroDict) )) + in_file = open( filePath, "r" ) + for line in in_file: + line = line.strip() + if line.startswith( '#' ) or len(line) == 0: + continue + if line.startswith( 'include' ) or line.startswith( '-include' ): + if line.startswith( '-include' ): + required = False + else: + required = True + includeFileRefs = line.split()[1:] + includeFiles = [] + # Expand macros and glob include file references + for ref in includeFileRefs: + ref = expandMacros( ref, macroDict ) + includeFiles += glob.glob( ref ) + # Recursively call getMacrosFromFile for each includeFile + for includeFile in includeFiles: + macroDict = getMacrosFromFile( includeFile, macroDict, debug, required ) + continue + + for regExp in [ macroNameRegExp, versionRegExp, epicsBaseVerRegExp ]: + macroMatch = regExp.search( line ) + if not macroMatch: + continue + macroName = macroMatch.group(1) + macroValue = macroMatch.group(2) + if macroName and macroValue: + if debug: + print("getMacrosFromFile: %s = %s" % ( macroName, macroValue )) + macroDict[ macroName ] = macroValue + break + + # Expand macro values + for macroName in macroDict: + macroValue = macroDict[macroName] + macroDict[macroName] = expandMacros( macroValue, macroDict ) + + if debug: + print("getMacrosFromFile %s: %d versions on exit" % ( filePath, len(macroDict) )) + return macroDict + +def getEpicsPkgDependents( topDir, debug=False ): + '''Find and return a dictionary of EPICS package (modules and base) versions + found in a release file. Ex. pkgDependents['base'] = 'R3.15.5-1.0' + ''' + macroDict = {} + macroDict['TOP'] = topDir + # Get the base and dependent modules from RELEASE files + releaseFile = os.path.join( topDir, "configure", "RELEASE" ) + if debug: + print("getEpicsPkgDependents: Checking release file: %s" % ( releaseFile )) + if os.path.isfile( releaseFile ): + macroDict = getMacrosFromFile( releaseFile, macroDict, debug=debug ) + + pkgDependents = {} + epicsModules = None + if 'EPICS_MODULES' in macroDict: + epicsModules = macroDict[ 'EPICS_MODULES' ] + for macroName in macroDict: + if macroName.endswith( '_MODULE_VERSION' ): + continue + macroValue = macroDict[macroName] + pkgName = macroNameToPkgName(macroName) + if not pkgName: + continue + if pkgName == 'base': + pkgVersion = os.path.split( macroValue )[-1] + elif epicsModules and macroValue.startswith( epicsModules ): + macroValue = macroValue.replace( epicsModules, '' ) + pkgVersion = os.path.split( macroValue )[-1] + else: + # Just show the last 4 levels of arbitrary paths:w + pkgVersion = '/'.join( macroValue.split('/')[-3:] ) + if pkgName and pkgVersion: + if debug: + print("getEpicsPkgDependents: %s = %s" % ( pkgName, pkgVersion )) + pkgDependents[ pkgName ] = pkgVersion + + return pkgDependents + +def pkgSpecToMacroVersions( pkgSpec, verbose=False ): + """ + Convert the pkgSpec into a dictionary of macroVersions + Each macroVersion entry maps macroName to version + """ + macroVersions = {} + ( pkgPath, pkgVersion ) = os.path.split( pkgSpec ) + pkgName = os.path.split( pkgPath )[1] + macroNames = pkgNameGetMacroNames( pkgName ) + for macroName in macroNames: + macroVersions[ macroName ] = pkgVersion + return macroVersions + +def update_pkg_dependency( topDir, pkgSpecs, debug=False, verbose=False ): + """ + update_pkg_dependency( + topDir, # path to top directory of epics package + pkgSpecs, # array of pkg specification strings: pkgPath/pkgVersion, ex asyn/R4.31 + verbose=False # show progress ) + Update the specified package dependencies, (module or base versions). + Checks and updates as needed: + TOP/RELEASE_SITE + TOP/configure/RELEASE + TOP/configure/RELEASE.local + Returns count of how many files were updated. + """ + # Check for a valid top directory + if not os.path.isdir( topDir ): + print("update_pkg_dependency: Invalid topDir: %s" % topDir) + return 0 + if verbose: + print("update_pkg_dependency: %s" % topDir) + + # Get current pkgSpecs + oldPkgDependents = getEpicsPkgDependents( topDir, debug=debug ) + oldMacroVersions = {} + for pkgName in oldPkgDependents: + pkgSpec = pkgName + "/" + oldPkgDependents[pkgName] + if verbose: + print("OLD: %s" % pkgSpec) + oldMacroVersions.update( pkgSpecToMacroVersions( pkgSpec ) ) + if len(oldMacroVersions) == 0: + print("update_pkg_dependency error: No pkgSpecs found under topDir:\n%s" % topDir) + return 0 + + # Convert the list of pkgSpecs into a list of macroVersions + # Each macroVersion is a tuple of ( macroName, version ) + newMacroVersions = {} + for pkgSpec in pkgSpecs: + if verbose: + print("NEW: %s" % pkgSpec) + newMacroVersions.update( pkgSpecToMacroVersions( pkgSpec ) ) + if len(newMacroVersions) == 0: + print("update_pkg_dependency error: No valid converions for pkgSpecs:") + print(pkgSpecs) + return 0 + + count = 0 + + for fileName in [ "RELEASE_SITE", + os.path.join( "configure", "RELEASE" ), + os.path.join( "configure", "RELEASE.local" ) ]: + filePath = os.path.join( topDir, fileName ) + if os.access( filePath, os.R_OK ): + count += update_pkg_dep_file( filePath, oldMacroVersions, newMacroVersions, verbose ) + return count + +# Check if any file inside configure/ has included a ../../RELEASE_SITE file +def hasIncludeDotDotReleaseSite(): + if not os.path.isdir( 'configure' ): + return False + # Just check configure/RELEASE and configure/RELEASE.local + for filename in [ 'RELEASE', 'RELEASE.local' ]: + configFilePath = os.path.join( 'configure', filename ) + if not os.path.isfile( configFilePath ): + continue + configFile = open( configFilePath, 'r') + for line in configFile: + # Check included ../../RELEASE_SITE file unless it is commented + if re.search('^[^\#]include(.*)/../../RELEASE_SITE', line): + return True + return False + +def doesPkgNeedMacro( macroName ): + ''' + Check if configure/RELEASE* files need a particular macro + ''' + if not macroName or len(macroName) == 0: + return False + # TODO: Check all configure/RELEASE* files + definesMacro = False + needsMacro = False + definesMacroRegExp = re.compile( '^%s\s*=\s*\S' % macroName ) + needsMacroRegExp = re.compile( '\$\(' + macroName ) + for filename in [ 'RELEASE', 'RELEASE.local' ]: + configFilePath = os.path.join( 'configure', filename ) + if not os.path.isfile( configFilePath ): + continue + configFile = open( configFilePath, 'r') + for line in configFile: + # Check if this macro is used + if needsMacroRegExp.search( line ): + needsMacro = True + # Check if this macro is defined + if definesMacroRegExp.search( line ): + definesMacro = True + + if needsMacro and definesMacro: + needsMacro = False + return needsMacro