update item DB phase 5

assets/database/leftover_db.json

@@ -1458,7 +1458,8 @@
 {"id":223186,"name":"Supply Expediter","icon":"ability_hunter_pathfinding","type":12,"requiresLevel":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"weaponSkills":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ilvl":1,"phase":3,"quality":3,"sources":[{"rep":{"repFactionId":2586,"repLevel":8}},{"rep":{"repFactionId":2587,"repLevel":8}}]},
 {"id":225838,"name":"Voltaic Icon","icon":"inv_relics_totemoflife","type":14,"rangedWeaponType":7,"requiresLevel":32,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"weaponSkills":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ilvl":1,"phase":4,"quality":2,"unique":true,"classAllowlist":[7],"sources":[{"drop":{"npcId":7158,"zoneId":361}}]},
 {"id":226546,"name":"Desert Sonar","icon":"inv_misc_enggizmos_20","type":12,"requiresLevel":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"weaponSkills":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ilvl":1,"phase":4,"quality":1,"unique":true},
-{"id":227688,"name":"Gorehowl","icon":"inv_axe_116","type":13,"weaponType":1,"handType":4,"requiresLevel":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"weaponSkills":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ilvl":1,"phase":4,"quality":1}
+{"id":227688,"name":"Gorehowl","icon":"inv_axe_116","type":13,"weaponType":1,"handType":4,"requiresLevel":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"weaponSkills":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ilvl":1,"phase":4,"quality":1},
+{"id":231015,"name":"Twill Mask","icon":"inv_misc_bandana_01","type":1,"armorType":1,"requiresLevel":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,7,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"weaponSkills":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"ilvl":10,"phase":5}
 ],
 "randomSuffixes":[
 ],
@@ -1582,8 +1583,8 @@
 {"effectId":1893,"spellId":20028,"name":"Enchant Chest - Major Mana","type":5,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,100,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},
 {"effectId":1894,"spellId":20029,"name":"Enchant Weapon - Icy Chill","type":13,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},
 {"effectId":1896,"spellId":20030,"name":"Enchant 2H Weapon - Superior Impact","type":13,"enchantType":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},
-{"effectId":1897,"spellId":20031,"name":"Enchant Weapon - Superior Striking","type":13,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},
 {"effectId":1897,"spellId":13695,"name":"Enchant 2H Weapon - Impact","type":13,"enchantType":1,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":1},
+{"effectId":1897,"spellId":20031,"name":"Enchant Weapon - Superior Striking","type":13,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},
 {"effectId":1898,"spellId":20032,"name":"Enchant Weapon - Lifestealing","type":13,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":3},
 {"effectId":1899,"spellId":20033,"name":"Enchant Weapon - Unholy Weapon","type":13,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},
 {"effectId":1900,"spellId":20034,"name":"Enchant Weapon - Crusader","type":13,"stats":[0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0,0],"quality":2},

assets/db_inputs/wowhead_spell_tooltips.csv

@@ -21579,6 +21579,7 @@
 453695,{"name":"Engrave Ring - Arcane Specialization","icon":"inv_jewelry_ring_31","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=453695/engrave-ring-arcane-specialization\"><b class=\"whtt-name\">Engrave Ring - Arcane Specialization</b></a><br />3 sec cast<br /><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td><span class=\"wowhead-tooltip-requirements\">Requires Ring</span><div class=\"q\">Engrave your ring with the Arcane Specialization rune:<br /><br /><span class=\"tooltip-inside-icon\" style=\"background-image: url(//wow.zamimg.com/images/wow/icons/small/inv_elemental_primal_mana.jpg)\"></span><br /><a href=\"/classic/spell=442893/arcane-specialization\" target=\"_blank\">Chance to hit with Arcane spells increased by 6%. Not cumulative with other ring runes.</a></div></td></tr></table>","tooltip2":"","buff":"","quality":-1,"spells":{},"buffspells":{},"completion_category":7}
 453696,{"name":"Engrave Ring - Fire Specialization","icon":"inv_jewelry_ring_31","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=453696/engrave-ring-fire-specialization\"><b class=\"whtt-name\">Engrave Ring - Fire Specialization</b></a><br />3 sec cast<br /><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td><span class=\"wowhead-tooltip-requirements\">Requires Ring</span><div class=\"q\">Engrave your ring with the Fire Specialization rune:<br /><br /><span class=\"tooltip-inside-icon\" style=\"background-image: url(//wow.zamimg.com/images/wow/icons/small/inv_elemental_primal_fire.jpg)\"></span><br /><a href=\"/classic/spell=442894/fire-specialization\" target=\"_blank\">Chance to hit with Fire spells increased by 6%. Not cumulative with other ring runes.</a></div></td></tr></table>","tooltip2":"","buff":"","quality":-1,"spells":{},"buffspells":{},"completion_category":7}
 453697,{"name":"Engrave Ring - Frost Specialization","icon":"inv_jewelry_ring_31","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=453697/engrave-ring-frost-specialization\"><b class=\"whtt-name\">Engrave Ring - Frost Specialization</b></a><br />3 sec cast<br /><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td><span class=\"wowhead-tooltip-requirements\">Requires Ring</span><div class=\"q\">Engrave your ring with the Frost Specialization rune:<br /><br /><span class=\"tooltip-inside-icon\" style=\"background-image: url(//wow.zamimg.com/images/wow/icons/small/inv_elemental_primal_water.jpg)\"></span><br /><a href=\"/classic/spell=442895/frost-specialization\" target=\"_blank\">Chance to hit with Frost spells increased by 6%. Not cumulative with other ring runes.</a></div></td></tr></table>","tooltip2":"","buff":"","quality":-1,"spells":{},"buffspells":{},"completion_category":7}
+457437,{"name":"Vanish","icon":"ability_vanish","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=457437/vanish\"><b class=\"whtt-name\">Vanish</b></a><table width=\"100%\"><tr><td>Instant</td><th>5 min cooldown</th></tr></table><div class=\"wowhead-tooltip-requirements\">Requires Rogue</div><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td>Reagents:<br /><div class=\"indent q1\"><a href=\"/classic/item=5140/flash-powder\">Flash Powder</a></div><div class=\"q\">Reduces all Magic damage taken by 50% for 10 sec.</div></td></tr></table>","tooltip2":"","buff":"<table><tr><td><b class=\"q\">Vanish</b></td></tr></table><table><tr><td>Magic damage taken reduced by 50%.<br><span class=\"q\">10 seconds remaining</span></td></tr></table>","quality":-1,"spells":{},"buffspells":{},"completion_category":0}
 458436,{"name":"Wyvern Strike","icon":"inv_spear_02","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=458436/wyvern-strike\"><b class=\"whtt-name\">Wyvern Strike</b></a><table width=\"100%\"><tr><td>55 Mana</td><th>Melee Range</th></tr></table><table width=\"100%\"><tr><td>Instant</td><th>8 sec cooldown</th></tr></table><div class=\"wowhead-tooltip-requirements\">Requires Hunter</div><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td><span class=\"wowhead-tooltip-requirements\">Requires Melee Weapon</span><div class=\"q\">A vicious strike that deals 100% weapon damage and causes the target to Bleed for (3 / 100 * 8 * Attack power) Physical damage over 8 sec.</div></td></tr></table>","tooltip2":"","buff":"<table><tr><td><b class=\"q\">Wyvern Strike</b></td></tr></table><table><tr><td>3 Bleed damage every 1 seconds.<br><span class=\"q\">8 seconds remaining</span></td></tr></table>","quality":-1,"spells":{},"buffspells":{},"completion_category":0}
 458481,{"name":"Wyvern Strike","icon":"inv_spear_02","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=458481/wyvern-strike\"><b class=\"whtt-name\">Wyvern Strike</b></a><table width=\"100%\"><tr><td>75 Mana</td><th>Melee Range</th></tr></table><table width=\"100%\"><tr><td>Instant</td><th>8 sec cooldown</th></tr></table><div class=\"wowhead-tooltip-requirements\">Requires Hunter</div><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td><span class=\"wowhead-tooltip-requirements\">Requires Melee Weapon</span><div class=\"q\">A vicious strike that deals 100% weapon damage and causes the target to Bleed for (4 / 100 * 8 * Attack power) Physical damage over 8 sec.</div></td></tr></table>","tooltip2":"","buff":"<table><tr><td><b class=\"q\">Wyvern Strike</b></td></tr></table><table><tr><td>4 Bleed damage every 1 seconds.<br><span class=\"q\">8 seconds remaining</span></td></tr></table>","quality":-1,"spells":{},"buffspells":{},"completion_category":0}
 458482,{"name":"Wyvern Strike","icon":"inv_spear_02","tooltip":"<table><tr><td><a class=\"whtt-name\" href=\"/classic/spell=458482/wyvern-strike\"><b class=\"whtt-name\">Wyvern Strike</b></a><table width=\"100%\"><tr><td>100 Mana</td><th>Melee Range</th></tr></table><table width=\"100%\"><tr><td>Instant</td><th>8 sec cooldown</th></tr></table><div class=\"wowhead-tooltip-requirements\">Requires Hunter</div><span class=\"wowhead-tooltip-requirements\">Requires level 1</span><br /></td></tr></table><table><tr><td><span class=\"wowhead-tooltip-requirements\">Requires Melee Weapon</span><div class=\"q\">A vicious strike that deals 100% weapon damage and causes the target to Bleed for (6 / 100 * 8 * Attack power) Physical damage over 8 sec.</div></td></tr></table>","tooltip2":"","buff":"<table><tr><td><b class=\"q\">Wyvern Strike</b></td></tr></table><table><tr><td>6 Bleed damage every 1 seconds.<br><span class=\"q\">8 seconds remaining</span></td></tr></table>","quality":-1,"spells":{},"buffspells":{},"completion_category":0}

docs/database_updates.md

@@ -0,0 +1,64 @@
+# Database Updates
+
+This doc outlines the various methods for updating database data in WoWSims Season of Discovery.
+
+## Introduction
+
+WoWSims uses a number of JSON database files to store our data.
+These can be found in [assets/database/](https://github.com/wowsims/sod/tree/master/assets/database):
+
+-   [db.json](https://github.com/wowsims/sod/blob/master/assets/database/db.json) - Our primary database file. This is where we store the majority of our item, spell, and other data used in the sims.
+-   [leftover_db.json](https://github.com/wowsims/sod/blob/master/assets/database/leftover_db.json) - This is a leftover database file where items filtered out for having invalid data (determined in `main.go`) are placed.
+
+We also store scraped data from external sources like Wowhead, Atlasloot, etc. in order to build database data in the first place.
+These can be found in [assets/db_inputs/](https://github.com/wowsims/sod/tree/master/assets/db_inputs):
+
+-   [atlasloot_db.json](https://github.com/wowsims/sod/blob/master/assets/db_inputs/atlasloot_db.json) - Scraped data from the AtlasLoot addon's GitHub repo. This is primarily used for item source data and faction restrictions.
+-   [wago_db2_items.csv](https://github.com/wowsims/sod/blob/master/assets/db_inputs/wago_db2_items.csv) - Scraped item data from Wago DB. This is currently used for additional item set and faction restriction information.
+-   [wowhead_gearplannerdb.txt](https://github.com/wowsims/sod/blob/master/assets/db_inputs/wowhead_gearplannerdb.txt) - Scraped item data from Wowhead Gear Planner. This is one of our primary sources of item data along with Wowhead item tooltips.
+-   [wowhead_item_tooltips.csv](https://github.com/wowsims/sod/blob/master/assets/db_inputs/wowhead_item_tooltips.csv) - Scraped item tooltips from Wowhead. We store the full tooltip along with the item IDs.
+-   [wowhead_rune_tooltips.csv](https://github.com/wowsims/sod/blob/master/assets/db_inputs/wowhead_rune_tooltips.csv) - Scraped rune tooltips from Wowhead. We store the full tooltip along with the rune engraving spell IDs.
+-   [wowhead_spell_tooltips.csv](https://github.com/wowsims/sod/blob/master/assets/db_inputs/wowhead_spell_tooltips.csv) - Scraped spell tooltips from Wowhead. We store the full tooltip along with the spell IDs.
+
+The entry point for running database scripts is [tools/database/gen_db/main.go](https://github.com/wowsims/sod/blob/master/tools/database/gen_db/main.go).
+This file is executed by running the `make items` command, or by running one of a set of commands listed in the comments of `gen_db/main.go`.
+
+-   `go run ./tools/database/gen_db -outDir=assets -gen=atlasloot`
+    -   Run the Atlasloot scraper. You do not need to delete existing data beforehand.
+-   `go run ./tools/database/gen_db -outDir=assets -gen=wago-db2-items`
+    -   Run the Wago DB2 scraper. You do not need to delete existing data beforehand.
+-   `go run ./tools/database/gen_db -outDir=assets -gen=wowhead-gearplannerdb`
+    -   Run the Wowhead Gear Planner scraper. You do not need to delete existing data beforehand.
+-   `go run ./tools/database/gen_db -outDir=assets -gen=wowhead-items`
+    -   Run the Wowhead Item Tooltips scraper. This only pulls in new entries, so if you want to regenerate existing data you will need to delete the existing entries. Often times you will want to pass IDs to restrict the range of items pulled in at once. This can be done using the `-id=`, `-minid=`, and `maxid=` commands.
+    -   Example: `go run ./tools/database/gen_db -outDir=assets -gen=wowhead-items -minid=200000 -maxid=235000`
+-   `go run ./tools/database/gen_db -outDir=assets -gen=wowhead-spells`
+    -   Run the Wowhead Spell Tooltips scraper. This only pulls in new entries, so if you want to regenerate existing data you will need to delete the existing entries. Often times you will want to pass IDs to restrict the range of items pulled in at once. This can be done using the `-id=`, `-minid=`, and `maxid=` commands.
+    -   Example: `go run ./tools/database/gen_db -outDir=assets -gen=wowhead-spells -minid=200000 -maxid=235000`
+-   `python3 tools/scrape_runes.py assets/db_inputs/wowhead_rune_tooltips.csv`
+    -   Run the Rune Tooltips scraper. This command is slightly different in that it actually visits Wowhead and parses data from the page using Selenium. You do not need to delete existing data beforehand.
+
+After performing any of these commands, you should then use `make items` in order to rebuild the database with the updated input data.
+
+## Overrides
+
+In addition to our db inputs, we can also define overrides for both adding and removing data.
+We have three different overrides files:
+
+-   [tools/database/overrides.go]https://github.com/wowsims/sod/blob/master/tools/database/overrides.go has several different types of overrides for items, including allowlists and denylists.
+-   [tools/database/enchant_overrides.go](https://github.com/wowsims/sod/blob/master/tools/database/enchant_overrides.go) has all of our enchant data and is where new enchant entries should be added
+-   [tools/database/rune_overrides.go](https://github.com/wowsims/sod/blob/master/tools/database/rune_overrides.go) has overrides related to runes, including overrides that fix some rune information scraped from wowhead, and a way to block runes from the sim until they can be implemented.
+
+These overrides should be used sparingly when possible, but are often a necessary part of filtering data in the sim's database.
+
+## Updating the Item Database
+
+With the frequent patch cadence of SoD we often need to pull in new and updated item data.
+This typically involves performing several commands to update the multiple sources of item data.
+
+1. Run the Wowhead Gear Planner scraper
+2. Run the Wowhead Item Tooltip scraper with the range of IDs you want to add/update, deleting any existing entries in that range if there are any.
+3. Optionally run the Atlasloot scraper to grab any new source data
+4. Finally run `make items` to regenerate the database.
+
+You should now also run tests because they could have changed if existing items were changed, then you're ready to commit everything.

sim/common/vanilla/item_effects.go

@@ -47,7 +47,7 @@ const (
 	ThrashBlade                    = 17705
 	SatyrsLash                     = 17752
 	MarkOfTheChosen                = 17774
-	Thunderfury                    = 19019
+	Thunderfury                    = 230224 // 19019
 	EbonHand                       = 19170
 	DarkmoonCardHeroism            = 19287
 	DarkmoonCardBlueDragon         = 19288

tools/database/overrides.go

@@ -91,8 +91,23 @@ var ItemOverrides = []*proto.UIItem{
 		{Source: &proto.UIItemSource_Quest{Quest: &proto.QuestSource{Id: 82083, Name: "A Broken Ritual"}}},
 	}},
 
-	// Heirloom Dwarven Handcannon, Wowhead partially glitchs out and shows us some other lvl calc for this
-	// {Id: 44093, Stats: stats.Stats{stats.MeleeCrit: 30, stats.SpellCrit: 30, stats.Resilience: 13, stats.AttackPower: 34}.ToFloatArray()},
+	// 2024-08-30 This item randomly vanished from Wowhead after Phase 5 datamining
+	{
+		Id:                 215161,
+		Name:               "Tempered Interference-Negating Helmet",
+		Icon:               "inv_helmet_49",
+		Type:               1,
+		ArmorType:          4,
+		RequiresLevel:      40,
+		Stats:              []float64{20, 0, 14, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 0, 0, 0, 0, 1, 0, 0, 0, 0, 0, 0, 426, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		WeaponSkills:       []float64{0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0},
+		Ilvl:               45,
+		Phase:              2,
+		Quality:            4,
+		RequiredProfession: 2,
+		Sources: []*proto.UIItemSource{
+			{Source: &proto.UIItemSource_Crafted{Crafted: &proto.CraftedSource{Profession: 2}}}},
+	},
 }
 
 // Keep these sorted by item ID.