Skip to content

Commit

Permalink
update atlasloot scraper, wowhead profession scraping, better profess…
Browse files Browse the repository at this point in the history 
…ion data display
  • Loading branch information
@kayla-glick
kayla-glick committed Dec 9, 2024
1 parent dc09a40 commit 3b6fc1f
Show file tree
Hide file tree
Showing 9 changed files with 3,586 additions and 3,273 deletions.
Binary file modified assets/database/db.bin
View file
Binary file not shown.
2,847 changes: 1,448 additions & 1,399 deletions assets/database/db.json
View file

Large diffs are not rendered by default.

Binary file modified assets/database/leftover_db.bin
View file
Binary file not shown.
180 changes: 90 additions & 90 deletions assets/database/leftover_db.json
View file

Large diffs are not rendered by default.

1,465 changes: 842 additions & 623 deletions assets/db_inputs/atlasloot_db.json
View file

Large diffs are not rendered by default.

2,200 changes: 1,104 additions & 1,096 deletions assets/db_inputs/wowhead_gearplannerdb.txt
View file

Large diffs are not rendered by default.

107 changes: 55 additions & 52 deletions tools/database/atlasloot.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -85,7 +85,8 @@ func readAtlasLootDungeonData(db *WowDatabase, expansion proto.Expansion, srcUrl
dungeonPattern := regexp.MustCompile(`data\["([^"]+)"] = {(.*?)items = {(.*?)@@@}@@@`)
mapIdRegexp := regexp.MustCompile(`MapID = (\d+),`)
npcNameAndIDPattern := regexp.MustCompile(`^[^@]*?AL\["(.*?)"\]\)?(.*?(@@@\s*npcID = {?(\d+),))?`)
diffItemsPattern := regexp.MustCompile(`\[([A-Z0-9]+_DIFF)\] = (({.*?@@@\s*},?@@@)|(.*?@@@\s*\),?@@@))`)
sodDiffItemsPattern := regexp.MustCompile(`\[(SOD_DIFF)\] = (({.*?@@@\s*},?@@@)|(.*?@@@\s*\),?@@@))`)
normalDiffItemsPattern := regexp.MustCompile(`\[(NORMAL_DIFF)\] = (({.*?@@@\s*},?@@@)|(.*?@@@\s*\),?@@@))`)
itemsPattern := regexp.MustCompile(`@@@\s+{(.*?)},`)
itemParamPattern := regexp.MustCompile(`AL\["(.*?)"\]`)

Expand Down Expand Up @@ -126,62 +127,68 @@ func readAtlasLootDungeonData(db *WowDatabase, expansion proto.Expansion, srcUrl
})
}

for _, difficultyMatch := range diffItemsPattern.FindAllStringSubmatch(npcSplit, -1) {
difficulty, ok := AtlasLootDifficulties[difficultyMatch[1]]
if !ok {
log.Fatalf("Invalid difficulty for NPC %s: %s", npcName, difficultyMatch[1])
}

curCategory := ""
curLocation := 0
// In AtlasLootClassic_SoD the maintainer split data into two categories: SOD_DIFF and NORMAL_DIFF.
// Any boss with loot changed in SoD will use SOD_DIFF, and if not it defaults to NORMAL_DIFF
var difficultyMatch []string
if sodDifficultyMatch := sodDiffItemsPattern.FindAllStringSubmatch(npcSplit, -1); len(sodDifficultyMatch) > 0 {
difficultyMatch = sodDifficultyMatch[0]
} else if normalDifficultyMatch := normalDiffItemsPattern.FindAllStringSubmatch(npcSplit, -1); len(normalDifficultyMatch) > 0 {
difficultyMatch = normalDifficultyMatch[0]
} else {
log.Fatalf("Invalid difficulty for NPC %s: %s", npcName, difficultyMatch[1])
}

for _, itemMatch := range itemsPattern.FindAllStringSubmatch(difficultyMatch[0], -1) {
itemParams := core.MapSlice(strings.Split(itemMatch[1], ","), strings.TrimSpace)
location, _ := strconv.Atoi(itemParams[0]) // Location within AtlasLoot's menu.
difficulty := proto.DungeonDifficulty_DifficultyNormal

idStr := itemParams[1]
if idStr[0] == 'n' || idStr[0] == '"' { // nil or "xxx"
if len(itemParams) > 3 {
if paramMatch := itemParamPattern.FindStringSubmatch(itemParams[3]); paramMatch != nil {
curCategory = paramMatch[1]
curLocation = location
}
}
if len(itemParams) > 4 {
if paramMatch := itemParamPattern.FindStringSubmatch(itemParams[4]); paramMatch != nil {
curCategory = paramMatch[1]
curLocation = location
}
}
} else { // item ID
itemID, _ := strconv.Atoi(idStr)
//fmt.Printf("Item: %d\n", itemID)
dropSource := &proto.DropSource{
Difficulty: difficulty,
}
curCategory := ""
curLocation := 0

if zoneID != 0 {
dropSource.ZoneId = int32(zoneID)
}
for _, itemMatch := range itemsPattern.FindAllStringSubmatch(difficultyMatch[0], -1) {
itemParams := core.MapSlice(strings.Split(itemMatch[1], ","), strings.TrimSpace)
location, _ := strconv.Atoi(itemParams[0]) // Location within AtlasLoot's menu.

if npcID == 0 {
dropSource.OtherName = npcName
} else {
dropSource.NpcId = int32(npcID)
idStr := itemParams[1]
if idStr[0] == 'n' || idStr[0] == '"' { // nil or "xxx"
if len(itemParams) > 3 {
if paramMatch := itemParamPattern.FindStringSubmatch(itemParams[3]); paramMatch != nil {
curCategory = paramMatch[1]
curLocation = location
}

if curCategory != "" && location == curLocation+1 {
}
if len(itemParams) > 4 {
if paramMatch := itemParamPattern.FindStringSubmatch(itemParams[4]); paramMatch != nil {
curCategory = paramMatch[1]
curLocation = location
dropSource.Category = curCategory
}
}
} else { // item ID
itemID, _ := strconv.Atoi(idStr)
//fmt.Printf("Item: %d\n", itemID)
dropSource := &proto.DropSource{
Difficulty: difficulty,
}

item := &proto.UIItem{Id: int32(itemID), Sources: []*proto.UIItemSource{{
Source: &proto.UIItemSource_Drop{
Drop: dropSource,
},
}}}
db.MergeItem(item)
if zoneID != 0 {
dropSource.ZoneId = int32(zoneID)
}

if npcID == 0 {
dropSource.OtherName = npcName
} else {
dropSource.NpcId = int32(npcID)
}

if curCategory != "" && location == curLocation+1 {
curLocation = location
dropSource.Category = curCategory
}

item := &proto.UIItem{Id: int32(itemID), Sources: []*proto.UIItemSource{{
Source: &proto.UIItemSource_Drop{
Drop: dropSource,
},
}}}
db.MergeItem(item)
}
}
}
Expand Down Expand Up @@ -397,10 +404,6 @@ var AtlasLootProfessionIDs = map[int]proto.Profession{
13: proto.Profession_Enchanting,
}

var AtlasLootDifficulties = map[string]proto.DungeonDifficulty{
"NORMAL_DIFF": proto.DungeonDifficulty_DifficultyNormal,
}

var AtlasLootPVPFactions = map[int]map[string]int32{
3277: {
// Silverwing Sentinels
Expand Down
29 changes: 21 additions & 8 deletions tools/database/wowhead_db.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -174,6 +174,7 @@ type WowheadItemSource struct {
C int32 `json:"c"`
Name string `json:"n"` // Name of crafting spell
Icon string `json:"icon"` // Icon corresponding to the named entity
SkillID int32 `json:"s"` // Skill ID
EntityID int32 `json:"ti"` // Crafting Spell ID / NPC ID / ?? / Quest ID
ZoneID int32 `json:"z"` // Only for drop / sold by sources
}
Expand All @@ -183,14 +184,14 @@ func (wi WowheadItem) ToProto() *proto.UIItem {
for i, details := range wi.SourceDetails {
switch wi.SourceTypes[i] {
case 1: // Crafted
// We'll get this from AtlasLoot instead because it can also tell us the profession.
//sources = append(sources, &proto.UIItemSource{
// Source: &proto.UIItemSource_Crafted{
// Crafted: &proto.CraftedSource{
// SpellId: details.EntityID,
// },
// },
//})
sources = append(sources, &proto.UIItemSource{
Source: &proto.UIItemSource_Crafted{
Crafted: &proto.CraftedSource{
Profession: WowheadProfessionIDs[details.SkillID],
SpellId: details.EntityID,
},
},
})
case 2: // Dropped by
sources = append(sources, &proto.UIItemSource{
Source: &proto.UIItemSource_Drop{
Expand Down Expand Up @@ -319,3 +320,15 @@ func (wi WowheadItem) getClassRestriction() []proto.Class {

return classAllowlist
}

var WowheadProfessionIDs = map[int32]proto.Profession{
//129: proto.Profession_FirstAid,
164: proto.Profession_Blacksmithing,
165: proto.Profession_Leatherworking,
171: proto.Profession_Alchemy,
//185: proto.Profession_Cooking,
186: proto.Profession_Mining,
197: proto.Profession_Tailoring,
202: proto.Profession_Engineering,
333: proto.Profession_Enchanting,
}
31 changes: 26 additions & 5 deletions ui/core/components/gear_picker/item_list.tsx
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,7 @@ import { EP_TOOLTIP } from '../../constants/tooltips';
import { setItemQualityCssClass } from '../../css_utils';
import { IndividualSimUI } from '../../individual_sim_ui';
import { Player } from '../../player';
import { Class, ItemQuality, ItemRandomSuffix, ItemSlot, ItemSpec } from '../../proto/common';
import { Class, ItemQuality, ItemRandomSuffix, ItemSlot, ItemSpec, Profession } from '../../proto/common';
import { DatabaseFilters, RepSource, UIEnchant, UIFaction, UIItem, UIItem_FactionRestriction, UIRune } from '../../proto/ui';
import { ActionId } from '../../proto_utils/action_id';
import { getUniqueEnchantString } from '../../proto_utils/enchants';
Expand Down Expand Up @@ -624,8 +624,8 @@ export default class ItemList<T extends ItemListType> {
const href = src.spellId ? ActionId.makeSpellUrl(src.spellId) : ActionId.makeItemUrl(item.id);
return makeAnchor(
href,
<div className="d-flex">
{this.getProfessionSourceIcon()}
<div className="d-flex align-items-center">
{this.getProfessionSourceIcon(src.profession)}
{professionNames.get(src.profession) ?? 'Unknown'}
</div>,
);
Expand Down Expand Up @@ -747,8 +747,29 @@ export default class ItemList<T extends ItemListType> {
return <img src="https://static.wikia.nocookie.net/wowpedia/images/1/1f/Pointer_buy_on_32x32.png" className="item-source-icon-vendor me-1" />;
}

private getProfessionSourceIcon(): Element {
return <img src="https://static.wikia.nocookie.net/wowpedia/images/6/63/Pointer_repair_off_32x32.png" className="item-source-icon-profession me-1" />;
private getProfessionSourceIcon(profession: Profession): Element {
let src = "https://static.wikia.nocookie.net/wowpedia/images/6/63/Pointer_repair_off_32x32.png"
switch (profession) {
case Profession.Alchemy:
src = "https://wow.zamimg.com/images/wow/icons/tiny/trade_alchemy.gif"
break;
case Profession.Blacksmithing:
src = "https://wow.zamimg.com/images/wow/icons/tiny/trade_blacksmithing.gif"
break;
case Profession.Enchanting:
src = "https://wow.zamimg.com/images/wow/icons/tiny/trade_enchanting.gif"
break;
case Profession.Engineering:
src = "https://wow.zamimg.com/images/wow/icons/tiny/trade_engineering.gif"
break;
case Profession.Leatherworking:
src = "https://wow.zamimg.com/images/wow/icons/tiny/trade_leatherworking.gif"
break;
case Profession.Tailoring:
src = "https://wow.zamimg.com/images/wow/icons/tiny/trade_tailoring.gif"
break;
}
return <img src={src} className="item-source-icon-profession me-1" />;
}

private getQuestSourceIcon(): Element {
Expand Down

0 comments on commit 3b6fc1f

Please sign in to comment.