Port improved reference tracking and harddel testing from tg. (BeeSta…

…tion#6274) * Enables 514 testing on master, Removes all reliance on extools outside of maptick (#56724) * Uses 514's map_cpu var when it's available * Uses auxtools for the debugger, to supply cross verison compatibility * Nukes extools reference tracking, reinstates the old ref tracking system (cherry picked from commit 40cdb6d025b02f7a5f845dc6984c1b05d48c4766) * Ref Tracking: Revengance (#57728) * Ref Tracking: Revengance Fixes reference tracking ignoring self references due to a poorly thought out tick checking system. Fixes reference tracking ignoring the contents of assoc lists Makes the reference tracking printouts actually describe what list the ref is in, rather then just saying "list" Adds REFERENCE_TRACKING_DEBUG, a define which toggles tracking info for the ref tracking procs, which allows for oversight on how the proc is working Allows for direct calls of qdel_and_find_ref_if_fail(), makes it use ref rather then REF(), fixing it breaking for mobs. (Ditto for the qdel hint which does the same thing) Moves REAGENTS_TESTING out of the reftracking define block Makes unit tests define REFERENCE_TRACKING, REFERENCE_TRACKING_DEBUG, and FIND_REF_NO_CHECK_TICK Adds a unit test that sanity checks the reference finder proc (cherry picked from commit 1aa42a318850f8ea32817931c38205dfd4bfd043) * Fixes some more holes in the ref tracker (#58972) * Fixes some more holes in the ref tracker The reference tracker was failing to check null keyed assoc list entries, along with being unable to check both lists in a list(list() = list()) pair This resolves that, and adds some new logic to the unit test to check for this sort of thing * Seperates the ref tracking unit test into 6 subtasks as requested (cherry picked from commit cfc75f5a848f1194e67ffe606ea42f4c9eacbb33) * Fixes a bunch of harddels that are sourced from player action (#59371) Sourced from #59118 and a cursed project I'll pr later, This pr contains a lot of harddel fixes for stuff that pops up after a player interacts with something. I'm not gonna list them all here because there's something like 60 130, check the commit log if you're curious Oh and I moved ref tracking screaming to a separate define, and made some optimizations to the thing in general. I think that's it, this pr is a bit of a frankenstine (cherry picked from commit f90e8cf7a3b35411fcdb3f582f0e4787929b55b6) * Put unit tests in a box, fixes sporradic gun test failure (#57408) * Initial commit * Fix removing old objects * Remove define * TGM (cherry picked from commit a5010312d9b188b93e886b5028ae6a8f0dde2bbc) * Del The World: Unit testing for hard deletes (#59612) Co-authored-by: SteelSlayer <[email protected]> (cherry picked from commit cd576ab519441fc2ac5cc1172aaec8d673baa3ca) * Fix typepath differences in exclusions * Remove missing typepaths from exclusions - separated for reference * Expose BAD_INIT defines * Ref tracker TLC, makes unit test harddel errors more descriptive (#62725) (cherry picked from commit 3bb64112e207c93f3874c82a4bfcc4955834fca7) * Harddel Fix Pack BeeStation#42 + Better Live Reftracking Support (#63877) * Hard Del Fixes, Ref Tracking Changes (cherry picked from commit f8aad14ae87c73edb7c4cba8644cbdd6977cd8d9) * Disable the harddel unit test. * Fix runtimes when finishing unit tests * Adds a fast path option for ref tracking * Fix review and add a task * Stop searching atoms after finding a reference Co-authored-by: ike709 <[email protected]> * Fixes a default warning in reference sanity. (#64560) (cherry picked from commit 79041601d49a7961bff26b0dd460a57c4770c9b1) Co-authored-by: LemonInTheDark <[email protected]> Co-authored-by: Mothblocks <[email protected]> Co-authored-by: ike709 <[email protected]> Co-authored-by: ike709 <[email protected]>
Darkmajia · Feb 4, 2022 · 402a7e9 · 402a7e9
1 parent ea6b4ca
commit 402a7e9
Show file tree

Hide file tree

Showing 24 changed files with 708 additions and 270 deletions.
diff --git a/.vscode/tasks.json b/.vscode/tasks.json
@@ -46,6 +46,19 @@
 			"group": "build",
 			"label": "Run All Tests"
 		},
+		{
+			"type": "shell",
+			"command": "bin/temp-fast-test",
+			"windows": {
+				"command": ".\\bin\\temp-fast-test.cmd"
+			},
+			"problemMatcher": [
+				"$tsc",
+				"$eslint-stylish"
+			],
+			"group": "build",
+			"label": "dm: find hard deletes"
+		},
 		{
 			"command": "${command:dreammaker.reparse}",
 			"group": "build",

diff --git a/_maps/templates/unit_tests.dmm b/_maps/templates/unit_tests.dmm
diff --git a/bin/temp-fast-test.cmd b/bin/temp-fast-test.cmd
@@ -0,0 +1,4 @@
+@echo off
+:: Nuke this task once the Del the World unit test passes. This exists to fix those errors easily.
+
+call "%~dp0\..\tools\build\build.bat" --wait-on-error dm-test -DREFERENCE_TRACKING_FAST %*
diff --git a/code/__DEFINES/is_helpers.dm b/code/__DEFINES/is_helpers.dm
@@ -9,6 +9,8 @@
 
 #define isatom(A) (isloc(A))
 
+#define isdatum(thing) (istype(thing, /datum))
+
 #define isweakref(D) (istype(D, /datum/weakref))
 
 //Turfs

diff --git a/code/__DEFINES/qdel.dm b/code/__DEFINES/qdel.dm
@@ -8,10 +8,10 @@
 #define QDEL_HINT_HARDDEL_NOW	4 //qdel should assume this object won't gc, and hard del it post haste.
 //defines for the gc_destroyed var
 
-#ifdef LEGACY_REFERENCE_TRACKING
-/** If LEGACY_REFERENCE_TRACKING is enabled, qdel will call this object's find_references() verb.
-  *
-  * Functionally identical to QDEL_HINT_QUEUE if GC_FAILURE_HARD_LOOKUP is not enabled in _compiler_options.dm.
+#ifdef REFERENCE_TRACKING
+/** If REFERENCE_TRACKING is enabled, qdel will call this object's find_references() verb.
+ *
+ * Functionally identical to [QDEL_HINT_QUEUE] if [GC_FAILURE_HARD_LOOKUP] is not enabled in _compiler_options.dm.
 */
 #define QDEL_HINT_FINDREFERENCE	5
 /// Behavior as QDEL_HINT_FINDREFERENCE, but only if the GC fails and a hard delete is forced.

diff --git a/code/__DEFINES/vv.dm b/code/__DEFINES/vv.dm
@@ -75,9 +75,6 @@
 #define VV_HK_MARK "mark"
 #define VV_HK_ADDCOMPONENT "addcomponent"
 #define VV_HK_MODIFY_TRAITS "modtraits"
-#ifdef REFERENCE_TRACKING
-#define VV_HK_VIEW_REFERENCES "viewreferences"
-#endif
 
 // /atom
 #define VV_HK_MODIFY_TRANSFORM "atom_transform"

diff --git a/code/__HELPERS/_logging.dm b/code/__HELPERS/_logging.dm
@@ -35,9 +35,16 @@
 	SEND_TEXT(world.log, text)
 #endif
 
-#ifdef REFERENCE_TRACKING_LOG
+#if defined(REFERENCE_DOING_IT_LIVE)
+#define log_reftracker(msg) log_harddel("## REF SEARCH [msg]")
+
+/proc/log_harddel(text)
+	WRITE_LOG(GLOB.harddel_log, text)
+
+#elif defined(REFERENCE_TRACKING) // Doing it locally
 #define log_reftracker(msg) log_world("## REF SEARCH [msg]")
-#else
+
+#else //Not tracking at all
 #define log_reftracker(msg)
 #endif
 

diff --git a/code/_compile_options.dm b/code/_compile_options.dm
@@ -11,26 +11,43 @@
 #ifdef TESTING
 #define DATUMVAR_DEBUGGING_MODE
 
-///Method of tracking references.
-//#define LEGACY_REFERENCE_TRACKING
-#ifdef LEGACY_REFERENCE_TRACKING
+///Used to find the sources of harddels, quite laggy, don't be surpised if it freezes your client for a good while
+//#define REFERENCE_TRACKING
+#ifdef REFERENCE_TRACKING
 
-///Should we be logging our findings or not
-#define REFERENCE_TRACKING_LOG
+///Used for doing dry runs of the reference finder, to test for feature completeness
+///Slightly slower, higher in memory. Just not optimal
+//#define REFERENCE_TRACKING_DEBUG
 
-///Use the legacy reference on things hard deleting by default.
+///Run a lookup on things hard deleting by default.
 //#define GC_FAILURE_HARD_LOOKUP
 #ifdef GC_FAILURE_HARD_LOOKUP
+///Don't stop when searching, go till you're totally done
 #define FIND_REF_NO_CHECK_TICK
 #endif //ifdef GC_FAILURE_HARD_LOOKUP
 
-#endif //ifdef LEGACY_REFERENCE_TRACKING
+#endif //ifdef REFERENCE_TRACKING
 
 
 //#define VISUALIZE_ACTIVE_TURFS	//Highlights atmos active turfs in green
 #endif //ifdef TESTING
 
-//#define UNIT_TESTS			//If this is uncommented, we do a single run though of the game setup and tear down process with unit tests in between
+/// If this is uncommented, we set up the ref tracker to be used in a live environment
+/// And to log events to [log_dir]/harddels.log
+//#define REFERENCE_DOING_IT_LIVE
+#ifdef REFERENCE_DOING_IT_LIVE
+// compile the backend
+#define REFERENCE_TRACKING
+// actually look for refs
+#define GC_FAILURE_HARD_LOOKUP
+#endif // REFERENCE_DOING_IT_LIVE
+
+#ifdef REFERENCE_TRACKING_FAST
+#define REFERENCE_TRACKING
+#define REFERENCE_TRACKING_DEBUG
+#endif
+
+//#define UNIT_TESTS //If this is uncommented, we do a single run though of the game setup and tear down process with unit tests in between
 
 #ifndef PRELOAD_RSC				//set to:
 #define PRELOAD_RSC	0			//	0 to allow using external resources or on-demand behaviour;
@@ -76,6 +93,14 @@
 #define TESTING
 #endif
 
+#if defined(UNIT_TESTS)
+//Hard del testing defines
+#define REFERENCE_TRACKING
+#define REFERENCE_TRACKING_DEBUG
+#define FIND_REF_NO_CHECK_TICK
+#define GC_FAILURE_HARD_LOOKUP
+#endif
+
 #ifdef TGS
 // TGS performs its own build of dm.exe, but includes a prepended TGS define.
 #define CBT

diff --git a/code/_globalvars/logging.dm b/code/_globalvars/logging.dm
@@ -78,3 +78,7 @@ GLOBAL_PROTECT(picture_logging_id)
 GLOBAL_VAR(picture_logging_prefix)
 GLOBAL_PROTECT(picture_logging_prefix)
 /////
+#ifdef REFERENCE_DOING_IT_LIVE
+GLOBAL_LIST_EMPTY(harddel_log)
+GLOBAL_PROTECT(harddel_log)
+#endif
diff --git a/code/controllers/subsystem/atoms.dm b/code/controllers/subsystem/atoms.dm
@@ -169,8 +169,3 @@ SUBSYSTEM_DEF(atoms)
 	var/initlog = InitLog()
 	if(initlog)
 		rustg_file_append(initlog, "[GLOB.log_directory]/initialize.log")
-
-#undef BAD_INIT_QDEL_BEFORE
-#undef BAD_INIT_DIDNT_INIT
-#undef BAD_INIT_SLEPT
-#undef BAD_INIT_NO_HINT
diff --git a/code/controllers/subsystem/garbage.dm b/code/controllers/subsystem/garbage.dm
@@ -24,8 +24,12 @@ SUBSYSTEM_DEF(garbage)
 
 	//Queue
 	var/list/queues
-	#ifdef LEGACY_REFERENCE_TRACKING
+	#ifdef REFERENCE_TRACKING
 	var/list/reference_find_on_fail = list()
+	#ifdef REFERENCE_TRACKING_DEBUG
+	//Should we save found refs. Used for unit testing
+	var/should_save_refs = FALSE
+	#endif
 	#endif
 
 /datum/controller/subsystem/garbage/get_metrics()
@@ -152,7 +156,7 @@ SUBSYSTEM_DEF(garbage)
 			++gcedlasttick
 			++totalgcs
 			pass_counts[level]++
-			#ifdef LEGACY_REFERENCE_TRACKING
+			#ifdef REFERENCE_TRACKING
 			reference_find_on_fail -= refID	//It's deleted we don't care anymore.
 			#endif
 			if (MC_TICK_CHECK)
@@ -169,14 +173,12 @@ SUBSYSTEM_DEF(garbage)
 		switch (level)
 			if (GC_QUEUE_CHECK)
 				#ifdef REFERENCE_TRACKING
-				D.find_references()
-				#elif defined(LEGACY_REFERENCE_TRACKING)
 				if(reference_find_on_fail[refID])
-					INVOKE_ASYNC(D, /datum/proc/find_references_legacy)
+					INVOKE_ASYNC(D, /datum/proc/find_references)
 					ref_searching = TRUE
 				#ifdef GC_FAILURE_HARD_LOOKUP
 				else
-					INVOKE_ASYNC(D, /datum/proc/find_references_legacy)
+					INVOKE_ASYNC(D, /datum/proc/find_references)
 					ref_searching = TRUE
 				#endif
 				reference_find_on_fail -= refID
@@ -192,10 +194,6 @@ SUBSYSTEM_DEF(garbage)
 						continue
 					to_chat(admin, "## TESTING: GC: -- [ADMIN_VV(D)] | [type] was unable to be GC'd --")
 				#endif
-				#ifdef REFERENCE_TRACKING
-				GLOB.deletion_failures += D //It should no longer be bothered by the GC, manual deletion only.
-				continue
-				#endif
 				I.failures++
 			if (GC_QUEUE_HARDDELETE)
 				HardDelete(D)
@@ -342,13 +340,13 @@ SUBSYSTEM_DEF(garbage)
 				SSgarbage.Queue(D, GC_QUEUE_HARDDELETE)
 			if (QDEL_HINT_HARDDEL_NOW)	//qdel should assume this object won't gc, and hard del it post haste.
 				SSgarbage.HardDelete(D)
-			#ifdef LEGACY_REFERENCE_TRACKING
-			if (QDEL_HINT_FINDREFERENCE) //qdel will, if LEGACY_REFERENCE_TRACKING is enabled, display all references to this object, then queue the object for deletion.
+			#ifdef REFERENCE_TRACKING
+			if (QDEL_HINT_FINDREFERENCE) //qdel will, if REFERENCE_TRACKING is enabled, display all references to this object, then queue the object for deletion.
 				SSgarbage.Queue(D)
-				D.find_references_legacy()
-			if (QDEL_HINT_IFFAIL_FINDREFERENCE)
+				D.find_references() //This breaks ci. Consider it insurance against somehow pring reftracking on accident
+			if (QDEL_HINT_IFFAIL_FINDREFERENCE) //qdel will, if REFERENCE_TRACKING is enabled and the object fails to collect, display all references to this object.
 				SSgarbage.Queue(D)
-				SSgarbage.reference_find_on_fail[REF(D)] = TRUE
+				SSgarbage.reference_find_on_fail["\ref[D]"] = TRUE
 			#endif
 			else
 				#ifdef TESTING

diff --git a/code/datums/datum.dm b/code/datums/datum.dm
@@ -41,10 +41,13 @@
 	*/
 	var/list/cooldowns
 
-
-#ifdef TESTING
+#ifdef REFERENCE_TRACKING
 	var/running_find_references
 	var/last_find_references = 0
+	#ifdef REFERENCE_TRACKING_DEBUG
+	///Stores info about where refs are found, used for sanity checks and testing
+	var/list/found_refs
+	#endif
 #endif
 
 #ifdef DATUMVAR_DEBUGGING_MODE
@@ -88,8 +91,13 @@
 			continue
 		qdel(timer)
 
-	//BEGIN: ECS SHIT
+	#ifdef REFERENCE_TRACKING
+	#ifdef REFERENCE_TRACKING_DEBUG
+	found_refs = null
+	#endif
+	#endif
 
+	//BEGIN: ECS SHIT
 	var/list/dc = datum_components
 	if(dc)
 		var/all_components = dc[/datum/component]

diff --git a/code/datums/datumvars.dm b/code/datums/datumvars.dm
@@ -31,9 +31,6 @@
 	VV_DROPDOWN_OPTION(VV_HK_EXPOSE, "Show VV To Player")
 	VV_DROPDOWN_OPTION(VV_HK_ADDCOMPONENT, "Add Component/Element")
 	VV_DROPDOWN_OPTION(VV_HK_MODIFY_TRAITS, "Modify Traits")
-	#ifdef REFERENCE_TRACKING
-	VV_DROPDOWN_OPTION(VV_HK_VIEW_REFERENCES, "View References")
-	#endif
 
 //This proc is only called if everything topic-wise is verified. The only verifications that should happen here is things like permission checks!
 //href_list is a reference, modifying it in these procs WILL change the rest of the proc in topic.dm of admin/view_variables!

diff --git a/code/game/objects/effects/landmarks.dm b/code/game/objects/effects/landmarks.dm
@@ -478,3 +478,13 @@ INITIALIZE_IMMEDIATE(/obj/effect/landmark/start/new_player)
 	GLOB.ruin_landmarks -= src
 	ruin_template = null
 	. = ..()
+
+/// Marks the bottom left of the testing zone.
+/// In landmarks.dm and not unit_test.dm so it is always active in the mapping tools.
+/obj/effect/landmark/unit_test_bottom_left
+	name = "unit test zone bottom left"
+
+/// Marks the top right of the testing zone.
+/// In landmarks.dm and not unit_test.dm so it is always active in the mapping tools.
+/obj/effect/landmark/unit_test_top_right
+	name = "unit test zone top right"
diff --git a/code/game/world.dm b/code/game/world.dm
@@ -18,6 +18,10 @@ GLOBAL_VAR(restart_counter)
 	GLOB.config_error_log = GLOB.world_manifest_log = GLOB.world_pda_log = GLOB.world_job_debug_log = GLOB.sql_error_log = GLOB.world_href_log = GLOB.world_runtime_log = GLOB.world_attack_log = GLOB.world_game_log = "data/logs/config_error.[GUID()].log" //temporary file used to record errors with loading config, moved to log directory once logging is set bl
 
 	config.Load(params[OVERRIDE_CONFIG_DIRECTORY_PARAMETER])
+
+	#ifdef REFERENCE_DOING_IT_LIVE
+	GLOB.harddel_log = GLOB.world_game_log
+	#endif
 
 	GLOB.revdata = new
 
@@ -123,6 +127,10 @@ GLOBAL_VAR(restart_counter)
 #ifdef UNIT_TESTS
 	GLOB.test_log = file("[GLOB.log_directory]/tests.log")
 	start_log(GLOB.test_log)
+#endif
+#ifdef REFERENCE_DOING_IT_LIVE
+	GLOB.harddel_log = "[GLOB.log_directory]/harddels.log"
+	start_log(GLOB.harddel_log)
 #endif
 	start_log(GLOB.world_game_log)
 	start_log(GLOB.world_attack_log)
-Original file line number
+Diff line change
@@ Expand Up / @@ -9,6 +9,8 @@ @@
     #define isatom(A) (isloc(A))
+    #define isdatum(thing) (istype(thing, /datum))
     #define isweakref(D) (istype(D, /datum/weakref))
     //Turfs
@@ Expand Down @@