Merge branch 'pre-release' into stable

MetPX · Aug 12, 2024 · d8d9452 · d8d9452
2 parents 7135b98 + c4f6c11
commit d8d9452
Show file tree

Hide file tree

Showing 21 changed files with 824 additions and 311 deletions.
diff --git a/BUILD.rst b/BUILD.rst
diff --git a/Makefile b/Makefile
@@ -22,6 +22,9 @@ SARRA_LINK = '-Wl,-rpath,$$ORIGIN/../lib' -L${SARRA_LIBDIR}
 
 CC = gcc
 
+VERSION = $(shell head -1 debian/changelog| sed 's/.*(//' | sed 's/).*//')
+MAJOR_VERSION = $(shell  echo "$(VERSION)" | sed 's+\..*++g' )
+
 # can also use intel compiller just by changing CC
 # CC = icc
 
@@ -37,29 +40,31 @@ LIBCLOCATION=$(shell ldd /bin/sh | awk '/libc\.so\./ { print; }' | cut -d' ' -f3
 # also remove -ljson-c from EXT_LIB declaration.
 # to work with sr3, change SR_APPNAME=\"sr3\" ... otherwise will be managed by version 2. 
 # on Power9, -fstack-check  causes coredumps, so removed for now.
+# On RHEL8, add -DINTERCEPT_SYSCALL to handle cases where mv calls syscall directly, instead of calling renameat2.
+# See sarrac issue #145 for more information about syscall/renameat2.
 
 CFLAGS = -DHAVE_JSONC -DSR_APPNAME=\"sr3\" -DFORCE_LIBC_REGEX=\"$(LIBCLOCATION)\" -fPIC -ftest-coverage -std=gnu99 -Wall -g -D_GNU_SOURCE $(RABBIT_INCDIR)
 
 SARRA_HEADER = sr_cache.h sr_config.h sr_consume.h sr_context.h sr_credentials.h sr_event.h sr_post.h sr_util.h sr_version.h uthash.h 
 SARRA_OBJECT = sr_post.o sr_consume.o sr_context.o sr_config.o sr_event.o sr_credentials.o sr_cache.o sr_util.o
-SARRA_LIB = libsr3c.so.1.0.0 
+SARRA_LIB = libsr3c.so.$(VERSION)
 EXT_LIB = -ljson-c -lrabbitmq -lcrypto -lc
-SHARED_LIB = libsr3shim.so.1 -o libsr3shim.so.1.0.0 libsr3shim.c libsr3c.so.1.0.0
+SHARED_LIB = libsr3shim.so.$(MAJOR_VERSION) -o libsr3shim.so.$(VERSION) libsr3shim.c libsr3c.so.$(VERSION)
 
 .c.o: $(SARRA_HEADER) Makefile
 	$(CC) $(CFLAGS) -c  $<
 
 #  head -1 debian/changelog | sed 's/.*(//' | sed 's/).*//'
 all: sr_version.h $(SARRA_OBJECT)
-	$(CC) $(CFLAGS) -shared -Wl,-soname,libsr3c.so.1 -o libsr3c.so.1.0.0 $(SARRA_OBJECT) -ldl $(RABBIT_LINK) $(EXT_LIB)
+	$(CC) $(CFLAGS) -shared -Wl,-soname,libsr3c.so.$(MAJOR_VERSION) -o libsr3c.so.$(VERSION) $(SARRA_OBJECT) -ldl $(RABBIT_LINK) $(EXT_LIB)
 	$(CC) $(CFLAGS) -shared -Wl,-soname,$(SHARED_LIB) -ldl $(SARRA_LINK) $(RABBIT_LINK) $(EXT_LIB)
 	if [ ! -f libsr3c.so ]; \
 	then \
-		ln -s libsr3c.so.1.0.0 libsr3c.so ; \
+		ln -s libsr3c.so.$(VERSION) libsr3c.so ; \
 	fi;
-	if [ ! -f libsr3c.so.1 ]; \
+	if [ ! -f libsr3c.so.$(MAJOR_VERSION) ]; \
 	then \
-		ln -s libsr3c.so.1.0.0 libsr3c.so.1 ; \
+		ln -s libsr3c.so.$(VERSION) libsr3c.so.$(MAJOR_VERSION) ; \
 	fi;
 	$(CC) $(CFLAGS) -o sr_configtest sr_configtest.c -lsr3c $(SARRA_LINK) -lrabbitmq $(RABBIT_LINK) -lcrypto
 	$(CC) $(CFLAGS) -o sr_utiltest sr_utiltest.c -lsr3c $(SARRA_LINK) -lrabbitmq $(RABBIT_LINK) -lcrypto
@@ -70,14 +75,15 @@ all: sr_version.h $(SARRA_OBJECT)
 #debian/changelog: ../sarracenia/debian/changelog
 #	sed 's/^metpx-sarracenia/libsarra-c/' <../sarracenia/debian/changelog >debian/changelog 
 
+
 sr_version.h: debian/changelog
-	echo "#define __sarra_version__ \"`head -1 debian/changelog| sed 's/.*(//' | sed 's/).*//'`\"" >sr_version.h
+	echo "#define __sarra_version__ \"$(VERSION)\"" >sr_version.h
 
 install:
 	@mkdir -p build build/bin build/lib build/include
 	@mv *.so build/lib
-	@mv *.so.1 build/lib
-	@mv *.so.*1.0.0 build/lib
+	@mv *.so.$(MAJOR_VERSION) build/lib
+	@mv *.so.*$(VERSION) build/lib
 	@mv sr3_cpost build/bin
 	@mv sr3_cpump build/bin
 	@cp *.h build/include/

diff --git a/README.rst b/README.rst
@@ -174,22 +174,6 @@ NOTE:
     The tests are disabled for now because the C consumer does lose messages.
 
 
-Source Code Documentation
--------------------------
-
-Partial implementation of Doxygen docs.
-to view::
-
-  sudo apt install doxygen
-  sudo apt install graphviz
-  doxygen
-
-and run, and it will create the browseable docs/html/ subdirectory
-with some documentation. Although only a few files out of the total
-have been properly annoted so far, doxygen is configured to extract
-as much information from the code as possible. (We have some pretty
-call/caller graphs now!)
-
 Limitations of the C implementation
 -----------------------------------
 
@@ -205,74 +189,6 @@ Limitations of the C implementation
  - The C implementation uses the libc(7) regular expression routines, which 
    are a little more limited than python ones.
 
-Build Dependencies
-------------------
-
-The librabbitmq version needs to be > 0.8,  this is newer than what is in
-ubuntu 16.04. So you need to git clone from https://github.com/alanxz/rabbitmq-c
-then built it there. The launchpad PPA has a backport included to take care of
-this::
-
-  export RABBIT_BUILD=*directory where rabbit has been built*
-
-
-librabbitmq-dev - AMQP client library written in C - Dev Files
-libssl-dev  - OpenSSL client library (used for hash algorithms.)
-libjson-c-dev - json-c header files.
-
-run dependencies::
-
-  librabbitmq4 - AMQP client library written in C
-  libssl - OpenSSL client library.
-  libjson-c3 - JSON manupulation library (for v03 parsing)
-
-On RPM-based distributions::
-  
-  librabbitmq-devel
-  json-c-devel
-
-This JSON library changed API multiple times in it's history. Sarrac development
-platform is ubuntu 18.04, where the libjson-c3 library is provided.  Older linux
-versions may have incompatible library versions and may not build or run correctly.
-The Makefile includes the -DHAVE_JSONC option in CFLAGS.  Removing this option
-disables v03 message parsing, but makes it possible to build on such systems.
-v03 posting will still work (dependency only required to receive messages.)
-
-BUILD OPTIONS
--------------
-
-FORCE_LIBC_REGEX
-~~~~~~~~~~~~~~~~
-
-This option is set by default as it is usually desired.
-If you see::
-
-  2018-11-21 00:08:17,315 [ERROR] invalid regular expression: .*\/tmp\/.*. Ignored
-
-and the regex is valid... the symptom we had was that the library was
-calling a version of the regular expresison routines included in a binary
-(ksh93 in this case) instead of the ones in libc that were expected.
-without this option being set, the shim library will compile and user
-Korn Shell regular expression grammar instead of the libc/posix ones.
-This is confusing in practice.
-
-Set the option::
-   
-   -DFORCE_LIBC_REGEX=\"/lib/x86_64-linux-gnu/libc.so.6\" 
-
-to the file containing the regcomp and regexec routines what are to be 
-used. The code uses dynamic library loading to force use of the specified
-routines. Obviously this setting is architecture dependent and would
-need adjustment if compiling on another platform, such as ARM or MIPS.
-
-SR_DEBUG_LOGS
-~~~~~~~~~~~~~
-
-To disable all log file support, so that diagnostics messages 
-are sent to standard error instead, include::
-
-  -DSR_DEBUG_LOGS=1
-
 
 Dorval Computing Centre
 -----------------------
@@ -289,100 +205,13 @@ To load sr3_cpost::
  
 or it may be installed in the system locations (/usr/bin, etc...)
 
+Build Process
+-------------
 
-Branches
---------
-
-The main repository of sarrac is: https://github.com/MetPX/sarrac
-As of 2022/12, development is on v3 compatible version of sarrac.
-The customer switched to sr3, so there is no need to continue with v2.
-The existing git branches and their roles:
-
-* development ... the default development branch, launchpad.net daily packages built from here.
-* pre-release ... v03 pre-release branch, tracks development punctually. launchpad.net pre-release packages built from here.
-* stable ... v03 release branch, tracks pre-release punctually. launchpad.net stable packages built from here.
-* v2 ... v2 development branch. (legacy)
-* v2_stable ... v2 release branch, tracks v2 punctually. (legacy)
-* issueXXX ... branch developed to address a particular issue.
+See `Building from Source <BUILD.rst>`_
 
 
 Release Process
 ---------------
 
-To note changes:
-  - Compile once with -DSR_DEBUG_LOGS to verify that no msg_log calls have errors (compiler will report)
-  - build package (without -DSR_DEBUG_LOGS) and install.
-  - run some ./test scripts.
-  - make trust_but_verify
-    examine results, which include a valgrind run.
-  - Run through a flow test.
-  - dch, and add your points.
-  - when ready to release, edit UNRELEASED to an appropriate status, usually unstable.
-  - git commit #what you need to commit...
-  - git tag <release> -m <release>
-  - git push
-  - git push origin <release>
-
-  - go to Launchpad, and import source `here <https://code.launchpad.net/~ssc-hpc-chp-spc/metpx-sarrac/+git/master>`_.
-  - go to launchpad, find the recipe and Request Build `here <https://code.launchpad.net/~ssc-hpc-chp-spc/+recipe/metpx-sarrac>`_.
-
-
-Building RPMS
--------------
-
-clone source to metpx-sr3c directory (needed by rpm build rules)::
-
-  git clone https://github.com/MetPX/sarrac metpx-sr3c
-  cd sr3c
-
-on Suse::
-
-  zypper addrepo https://download.opensuse.org/repositories/network:messaging:amqp/openSUSE_Leap_15.1/network:messaging:amqp.repo
-  zypper refresh
-  zypper install librabbitmq4 librabbitmq-devel
-  zypper install libopenssl-devel libjson-c-devel
-  make rpm_suse15
-
-on Redhat/Centos::
-
-  make rpm_rhel7
-
-
-( notes from: https://github.com/MetPX/sarrac/issues/73 )
-
-
-Proposed Coding Style Guidelines
---------------------------------
-
-Generally, we used `Linux Kernel C Guidelines <https://www.kernel.org/doc/html/v4.10/process/coding-style.html>`_
-
-but with one pythonic affectation:  You know Rule 1? about the tabs with 8 characters?  This code base is normally
-indented like python instead, use four spaces, and not tabs.
-
-Also, in terms of documenting code, when adding new things one should add comments
-keeping in minde compatbility with `doxygen <http://www.doxygen.nl/manual/docblocks.html>`_ 
-Examples::
-
-  /**
-   *  descriptive comment above a struct or function declaration.
-   */ 
-
-  /**
-   * sr_config_find_one() - find the name configuration file name 
-   * \param sr_cfg:       The configuration to be updated with the configuration found.
-   * \param original_one: The name provided by the user.
-   *
-   * Return pointer to the matching mask if there is one, NULL otherwise.
-   * The pointer to char will hold the absolute path of the config file corresponding to original_one
-   *
-   * Return: pointer to a static char buffer with a path name to the corresponding configuration file.
-   */
-
-  char foo; /**< description of foo class member */
-
-
-The code has a mix of comments is not entirely doxygen friendly just yet.  Feel free
-to improve.  Other than that... the kernel C guidelines are the rule.
-
-FIXME: We ran a code reformatter on it once... should probably repeat from time to time, would be 
-useful to document which one was used. I believe it undoes for the pythonic exception.
+See `Releases <Release.rst>`_
diff --git a/RELEASE.rst b/RELEASE.rst
@@ -0,0 +1,98 @@
+
+------------------------------
+Making a Release of MetPX-sr3c
+------------------------------
+
+Release Targets:
+
+* hpc: intel and powerpc
+* launchpad: whatever platforms it builds for (ARM would be cool.)
+* upload packages to github, which acts as central node for release notifications.
+
+* See `Build <Build.rst>`_ for instructions on local building.
+
+Pre-Rrelease QA/Validation
+--------------------------
+
+Finalize the changes:
+
+  - update debian/changelog appropriately for the release. 
+  - in debian/changelog, Set the version on the first line correctly. This version will determine the label on the built packages, regardless of git tag. (perhaps use dch on debian/ubuntu system.)
+  - Compile once with -DSR_DEBUG_LOGS to verify that no msg_log calls have errors (compiler will report)
+  - make trust_but_verify
+    examine results, which include a valgrind run.
+
+Once that is clean, proceed to the Pre-Release Porting:
+
+Pre-Release Porting
+~~~~~~~~~~~~~~~~~~~
+
+Development is mostly done on new operating systems. The most important platforms
+for deployment, however, are pretty old.
+
+Prior to doing a full release, should install on all platforms and run tests. on Github,
+these tests are done by Actions automatically, so one just has to review results for
+ubuntu 20.04, 22.04, and 24.04.  However one must run the tests manually on the following
+platforms:
+
+for each of:
+
+* redhat8 
+* redhat9
+* ubuntu18
+
+do:
+
+  * https://github.com/MetPX/sarrac/blob/development/BUILD.rst#testing
+  * then install the package locally: either
+
+     * https://github.com/MetPX/sarrac/blob/development/BUILD.rst#build-a-debian-package  or 
+     * https://github.com/MetPX/sarrac/blob/development/BUILD.rst#build-a-debian-package
+
+  * install them... 
+
+     * rpm -ivh or dpkg -i ...
+
+  * then run the python flow_tests. as per the python package documentation:
+
+    * https://metpx.github.io/sarracenia/Contribution/Release.html#pre-release-overview
+
+
+Release Process
+---------------
+
+To note changes:
+
+  - install package so that it is used for flow tests on at least one platform.
+  - find redhat8 (with local disk) and run make test_shim
+
+  - on redhat8, edit Makefile to add -DINTERCEPT_SYSCALL to CFLAGS.
+
+  - find ubuntu18 (with local disk) and run make test_shim.
+  - dch, and touch up your points if need be.
+  - when ready to release, edit UNRELEASED to an appropriate status, usually unstable.
+  - git commit #what you need to commit...
+  - git tag <release> -m <release>
+  - git push
+  - git push origin <release>
+
+  - go to Launchpad, and import source `here <https://code.launchpad.net/~ssc-hpc-chp-spc/metpx-sarrac/+git/master>`_.
+  - go to launchpad, find the recipe and Request Build `here <https://code.launchpad.net/~ssc-hpc-chp-spc/+recipe/metpx-sr3c-pre-release>`_.
+  - go to an hpc account (on an intel node)
+    * mkdir ~/Sarracenia;  cd ~/Sarracenia
+    * git clone https://github.com/MetPX/sarrac metpx-sr3c
+    * edit Makefile to add -DINTERCEPT_SYSCALL to CFLAGS.
+    * cd metpx-sr3c
+    * make rpm_rhel7 
+    * rpm -ivh ~/rpmbuild/RPMS/_platform_/*version*.rpm  (if upgrading, -Uvh)
+
+  - go to a hpc account on a powerpc node
+    * cd Sarracenia/metpx-sr3c
+    * make clean
+    * make rpm_rhel7
+    * rpm -ivh ~/rpmbuild/RPMS/_platform_/*version*.rpm
+
+The rpm build targets run *rpmbuild* which places the packages in a standard tree in the user account:
+~/rpmbuild/RPMS/<arch>/..rpm on each arch the rpm will be created in the appropriate directory.
+
+
diff --git a/debian/changelog b/debian/changelog
@@ -1,13 +1,19 @@
-metpx-sr3c (3.24.02) unstable; urgency=medium
+metpx-sr3c (3.24.07) unstable; urgency=medium
 
+  * fix #154 rename ignored when one end is outside tree being watched with inotify.
+  * support #147 coreutils using syscall instead of renameat2 on redhat8 and
+    ubuntu 18.
+  * fix #148 sr3_cpost posting messages with corrupt topics if relPath blank.
   * fix #141 sr3_cpost in a tree being mirrored (conflict with shim)
   * tests: fix for link modification times cannot be preserved.
   * test fixes for changes in python side.
   * README revised to reflect that cpump is not resilient to failure.
   * fix #133 read some non-utf8 fields (instead of erroring.)
   * last tx/rx should be cumulative, reset less often.
+  * fix SR_DEBUG_LOGS being broken by addition of context to api signature
+  * tests working on RedHat 8 and 9
 
- -- peter <[email protected]>  Tue, 15 Jan 2024 08:26:22 -0500
+ -- Reid Sunderland <[email protected]>  Mon, 12 Aug 2024 15:06:22 -0600
 
 metpx-sr3c (3.23.11p3) unstable; urgency=medium