Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

TIND ILS: Add 269, 720 fields #3413

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
+373 −137
Open

TIND ILS: Add 269, 720 fields #3413

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
c07c91b
MARC: Map 720 as a contributor
skairunner Jan 28, 2025
f58876b
MARCXML: Refactor to expose a utility.
skairunner Jan 29, 2025
451325a
TIND ILS: Use 269 for date
skairunner Jan 29, 2025
257dfde
lint
skairunner Feb 3, 2025
5756a08
Fix MARC.js suggestions
skairunner Feb 4, 2025
5dfd1b8
MARCXML: Update tests
skairunner Feb 4, 2025
3385cb3
TIND ILS: Reflect comments
skairunner Feb 4, 2025
23a02ff
MARC: Apply author processing to 720 field
skairunner Feb 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
151 changes: 121 additions & 30 deletions Library Catalog (TIND ILS).js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -9,14 +9,14 @@
"inRepository": true,
"translatorType": 4,
"browserSupport": "gcsibv",
"lastUpdated": "2025-01-29 18:53:58"
"lastUpdated": "2025-02-04 07:37:55"
}

/*
***** BEGIN LICENSE BLOCK *****

Copyright © 2021 Abe Jellinek

This file is part of Zotero.

Zotero is free software: you can redistribute it and/or modify
Expand Down Expand Up @@ -56,7 +56,7 @@ function detectWeb(doc, url) {
if (!doc.querySelector('#tindfooter')) {
return false;
}

if (url.includes('/record/')) {
const schemaOrg = getSchemaOrg(doc);

Expand Down Expand Up @@ -93,49 +93,65 @@ function getSearchResults(doc, checkOnly) {
return found ? items : false;
}

function doWeb(doc, url) {
async function doWeb(doc, url) {
if (detectWeb(doc, url) == "multiple") {
Zotero.selectItems(getSearchResults(doc, false), function (items) {
if (items) ZU.processDocuments(Object.keys(items), scrape);
});
}
else {
scrape(doc, url);
await scrape(doc, url);
}
}

/**
*
* @param {Document} doc The page document
*/
function scrape(doc, _url) {
async function scrape(doc, _url) {
let translator = Zotero.loadTranslator("import");
// MARCXML
translator.setTranslator("edd87d07-9194-42f8-b2ad-997c4c7deefd");

const schemaOrg = getSchemaOrg(doc);

let marcXMLURL = attr(doc, 'a[href$="/export/xm"], a[download$=".xml"]', 'href');
ZU.doGet(marcXMLURL, function (respText) {
var translator = Zotero.loadTranslator("import");
// MARCXML
translator.setTranslator("edd87d07-9194-42f8-b2ad-997c4c7deefd");
translator.setString(respText);

translator.setHandler("itemDone", function (obj, item) {
item.libraryCatalog = text(doc, '#headerlogo')
|| attr(doc, 'meta[property="og:site_name"]', 'content');

let erURL = attr(doc, '.er-link', 'href');
if (erURL) {
item.url = erURL;
}

if (schemaOrg) {
enrichItemWithSchemaOrgItemType(item, schemaOrg);
}
let [respText, marcxml] = await Promise.all([
requestText(marcXMLURL),
translator.getTranslatorObject(),
]);

item.complete();
});
const xml = new DOMParser().parseFromString(respText, "text/xml")

for await (let record of marcxml.parseDocument(xml)) {
let item = new Zotero.Item();

translator.translate();
});
// Set the right item type if schemaorg exists
if (schemaOrg) {
enrichItemWithSchemaOrgItemType(item, schemaOrg);
}

record.translate(item);
//// Enhance the item with TIND-specific information
// Catalog name
item.libraryCatalog = text(doc, '#headerlogo')
|| attr(doc, 'meta[property="og:site_name"]', 'content');

// Url
let erURL = attr(doc, '.er-link', 'href');
if (erURL) {
item.url = erURL;
}

// Tind-specific date field 269. Assume there's only one.
let tindDate = record.getFieldSubfields("269")[0];
if (tindDate && tindDate.a && !item.hasOwnProperty("date")) {
// Assign the date if it doesn't already exist.
item.date = tindDate.a;
}

item.complete();
}
}

/**
Expand Down Expand Up @@ -375,7 +391,7 @@ var testCases = [
},
{
"type": "web",
"url": "https://library.usi.edu/record/1416599",
"url": "https://library.usi.edu/record/1416599?v=pdf",
"items": [
{
"itemType": "thesis",
Expand All @@ -385,8 +401,24 @@ var testCases = [
"firstName": "Sherry",
"lastName": "Crawford",
"creatorType": "author"
},
{
"lastName": "Wilhelmus, Thomas A",
skairunner marked this conversation as resolved.
Show resolved Hide resolved
"creatorType": "contributor",
"fieldMode": 1
},
{
"lastName": "Sands, Helen R",
"creatorType": "contributor",
"fieldMode": 1
},
{
"lastName": "Musgrove, Laurence E",
"creatorType": "contributor",
"fieldMode": 1
}
],
"date": "1996",
"abstractNote": "No abstract",
"language": "eng",
"libraryCatalog": "University of Southern Indiana",
Expand Down Expand Up @@ -458,14 +490,73 @@ var testCases = [
}
]
},
{
"type": "web",
"url": "https://socialmediaarchive.org/record/70",
"items": [
{
"itemType": "dataset",
"title": "Diffusion Time Metrics for Facebook Posts with 100 or More Reshares",
"creators": [
{
"lastName": "Meta Platforms, Inc",
"creatorType": "contributor",
"fieldMode": 1
}
],
"abstractNote": "This dataset contains aggregated information about all content reshared 100 or more times from July 1, 2020 through February 1, 2021. Each row of the dataset corresponds to an individual tree and its size and depth at specific hours and days from initial posting",
"libraryCatalog": "Social Media Archive at ICPSR - SOMAR",
"attachments": [],
"tags": [
{
"tag": "United States"
},
{
"tag": "elections"
},
{
"tag": "political attitudes"
},
{
"tag": "political behavior"
},
{
"tag": "social media"
},
{
"tag": "web platform data"
}
],
"notes": [
{
"note": "The U.S. 2020 Facebook and Instagram Election Study (US 2020 FIES) is a partnership between Meta and academic researchers to understand the impact of Facebook and Instagram on key political attitudes and behaviors during the US 2020 election"
}
],
"seeAlso": [],
"date": "2024-12-10"
}
]
},
{
"type": "web",
"url": "https://socialmediaarchive.org/record/60",
"items": [
{
"itemType": "dataset",
"title": "Not With a Bang But a Tweet: Democracy, Culture Wars, and the Memeification of T.S. Eliot",
"creators": [],
"creators": [
{
"lastName": "Walsh, Melanie",
"creatorType": "editor",
"fieldMode": 1
},
{
"lastName": "Preus, Anna",
"creatorType": "editor",
"fieldMode": 1
}
],
"date": "2024-10-04",
"abstractNote": "This dataset includes posts from Twitter (now X) from 2006 to early 2022 that mentioned a variation of T.S. Eliot's famous lines \"This is the way the world ends / Not with a bang but a whimper\" (see \"Design\" for specific search terms used).\n<br><br>\nModernist poet T.S. Eliot concluded his 1925 poem \"The Hollow Men\" with the iconic lines: \"This is the way the world ends / Not with a bang but a whimper.\" When Eliot died in 1965, the New York Times claimed in his obituary that these lines were “probably the most quoted lines of any 20th-century poet writing in English.” They may be among the most memed lines, as well. Through a computational analysis of Twitter data, we have found that at least 350,000 tweets have referenced or remixed Eliot’s lines since the beginning of Twitter’s history in 2006. While references to the poem vary widely, we focus on two prominent political usages of the phrase — cases where Twitter users invoke it to warn about the state of modern democracy, often from the left side of the political spectrum, and cases where they use the phrase to critique political correctness and “cancel culture” or to mock people for non-normatized aspects of their identities, often from the right side of the political spectrum. Though some of the tweets cite Eliot directly, most do not, and in many cases the phrase almost seems to be moving from an authored quotation into a common idiom or turn-of-phrase. Linguistics experts increasingly refer to this kind of construction as a “snowclone” —a fixed phrasal template, often with a culturally salient source (e.g., a quotation from a book, TV show, or movie), that has “one or more variable slots” into which users insert various “lexical substitutions\" (Hartmann and Ungerer). This data thus enables researchers to study both the circulation of literature and the evolution of linguistic forms",
"libraryCatalog": "Social Media Archive at ICPSR - SOMAR",
"shortTitle": "Not With a Bang But a Tweet",
Expand Down
119 changes: 85 additions & 34 deletions MARC.js
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@
"priority": 100,
"inRepository": true,
"translatorType": 1,
"lastUpdated": "2024-03-15 21:14:09"
"lastUpdated": "2025-02-05 08:11:17"
}

/*
Expand Down Expand Up @@ -344,9 +344,11 @@ record.prototype._associateTags = function (item, fieldNo, part) {

// this function loads a MARC record into our database
record.prototype.translate = function (item) {
// get item type
if (this.leader) {
var marcType = this.leader.substr(6, 1);
// Set item type if it does not already exist.
// This is because some translators (e.g. TIND ILS) have non-MARC ways to determine item type,
// but the MARC translator can return incorrect results for the creators if the item type is book.
if (!item.itemType && this.leader) {
let marcType = this.leader.substr(6, 1);
if (marcType == "g") {
item.itemType = "film";
}
Expand All @@ -363,11 +365,8 @@ record.prototype.translate = function (item) {
// 20091210: in unimarc, the code for manuscript is b, unused in marc21.
item.itemType = "manuscript";
}
else {
item.itemType = "book";
}
}
else {
if (!item.itemType) {
item.itemType = "book";
}

Expand Down Expand Up @@ -516,36 +515,43 @@ record.prototype.translate = function (item) {
trl: "translator"
};

var creatorFields = ["100", "110", "700", "710"];// "111", "711" are meeting name
for (let i = 0; i < creatorFields.length; i++) {
var authorTab = this.getFieldSubfields(creatorFields[i]);
let creatorFields = ["100", "110", "700", "710", "720"];// "111", "711" are meeting name
for (let tag of creatorFields) {
let authorTab = this.getFieldSubfields(tag);
for (let j in authorTab) {
if (authorTab[j]['4'] && RELATERM[authorTab[j]['4']] && RELATERM[authorTab[j]['4']] == "SKIP") {
let datafield = authorTab[j];

// Skip if there is no $a subfield, aka a name
if (!datafield.a) {
continue;
}
var creatorObject = {};
if (authorTab[j].a) {
if (creatorFields[i] == "100" || creatorFields[i] == "700") {
creatorObject = ZU.cleanAuthor(authorTab[j].a, "author", true);
}
else {
// same replacements as in the function ZU.cleanAuthor for institutional authors:
authorTab[j].a = authorTab[j].a.replace(/^[\s\u00A0.,/[\]:]+/, '')
.replace(/[\s\u00A0.,/[\]:]+$/, '')
.replace(/[\s\u00A0]+/, ' ');
creatorObject = { lastName: authorTab[j].a, creatorType: "contributor", fieldMode: 1 };
}
// some heuristic for the default values:
// in a book without any person as a main entry (no 100 field)
// it is likely that all persons (in 700 fields) are editors
if (creatorFields[i] == "700" && !this.getFieldSubfields("100")[0] && item.itemType == "book") {
creatorObject.creatorType = "editor";
}
if (authorTab[j]['4'] && RELATERM[authorTab[j]['4']]) {
creatorObject.creatorType = RELATERM[authorTab[j]['4']];
}
item.creators.push(creatorObject);

// Skip "SKIP" fields
if (datafield['4'] && RELATERM[datafield['4']] && RELATERM[datafield['4']] == "SKIP") {
continue;
}

if (tag == "100" || tag == "700") {
creatorObject = ZU.cleanAuthor(datafield.a, "author", true);
} else if (tag == "720") {
creatorObject = ZU.cleanAuthor(datafield.a, "contributor", true);
} else {
// same replacements as in the function ZU.cleanAuthor for institutional authors:
datafield.a = datafield.a.replace(/^[\s\u00A0.,/[\]:]+/, '')
.replace(/[\s\u00A0.,/[\]:]+$/, '')
.replace(/[\s\u00A0]+/, ' ');
creatorObject = { lastName: datafield.a, creatorType: "contributor", fieldMode: 1 };
}
// some heuristic for the default values:
// in a book without any person as a main entry (no 100 field)
// it is likely that all persons (in 700 fields) are editors
if (tag == "700" && !this.getFieldSubfields("100")[0] && item.itemType == "book") {
creatorObject.creatorType = "editor";
}
if (datafield['4'] && RELATERM[datafield['4']]) {
creatorObject.creatorType = RELATERM[datafield['4']];
}
item.creators.push(creatorObject);
}
}

Expand Down Expand Up @@ -1023,6 +1029,51 @@ var testCases = [
"seeAlso": []
}
]
},
{
"type": "import",
"input": "03613 a2200469 4500001000300000005001700003024004300020037001000063245007200073251000700145269001500152336001200167500024700179510021700426520026800643536002500911540051800936650001701454650002401471650002301495650001401518651001801532655002201550720005701572791014401629856013801773856019701911856017102108906002702279908001302306910002702319912022802346914007202574924022402646926000602870927013602876930006803012980001303080980001303093980001103106981002603117\u001e70\u001e20241218140557.0\u001e70\u001fahttps://doi.org/10.3886/qkqw-yg74\u001f2DOI\u001e \u001faADMIN\u001e \u001faDiffusion Time Metrics for Facebook Posts with 100 or More Reshares\u001e \u001fav1\u001e \u001fa2024-12-10\u001e \u001faDataset\u001e \u001faThe U.S. 2020 Facebook and Instagram Election Study (US 2020 FIES) is a partnership between Meta and academic researchers to understand the impact of Facebook and Instagram on key political attitudes and behaviors during the US 2020 election.\u001e \u001faMeta Platforms, Inc. Diffusion Time Metrics for Facebook Posts with 100 or More Reshares. Inter-university Consortium for Political and Social Research [distributor], 2024-12-10. https://doi.org/10.3886/qkqw-yg74\u001e \u001faThis dataset contains aggregated information about all content reshared 100 or more times from July 1, 2020 through February 1, 2021. Each row of the dataset corresponds to an individual tree and its size and depth at specific hours and days from initial posting.\u001e \u001foMeta Platforms, Inc.\u001e \u001faThis dataset has two levels of access: Public and Restricted.\n<ul>\n<li>Public files can be downloaded directly from the dataset record. Typically, documentation files such as READMEs and codebooks are made public.</li>\n\n<li>Restricted data files require a Restricted Data Application and will be accessed through a secure virtual data enclave. <a href=\"https://docs.google.com/document/d/1NcohZjrhh_F_GpbdArI7DvHBB1adZ80ojFzhy7zvBM4/edit?usp=sharing \">Learn more about applying for restricted data.</a></li>\n</ul>\u001e \u001fasocial media\u001e \u001fapolitical attitudes\u001e \u001fapolitical behavior\u001e \u001faelections\u001e \u001fzUnited States\u001e \u001faweb platform data\u001e \u001faMeta Platforms, Inc.\u001feData Collector\u001f7Organizational\u001e \u001ftThe Diffusion and Reach of (Mis)Information on Facebook during the US 2020 Election\u001faJournal Article\u001fwhttps://doi.org/10.15195/v11.a41\u001f2DOI\u001e4 \u001fuhttps://socialmediaarchive.org/record/70/files/US2020_Glossary_v1.csv\u001fyGlossary of terms\u001f98a2f2464-fb91-47f4-8bb1-30943b39ae53\u001fs37282\u001e4 \u001fuhttps://socialmediaarchive.org/record/70/files/data_dictionary_diffusion_time_metrics_facebook_posts_with_100_or_more_reshares.csv\u001fyData dictionary\u001f9b7b7b1e4-b130-4671-af00-1d885d273132\u001fs32034\u001e4 \u001fuhttps://socialmediaarchive.org/record/70/files/US2020_FB%26IG_Elections_External_Codebook_v3.pdf\u001fyProject-level codebook\u001f984602a60-15eb-4955-ae57-feaa347b1699\u001fs848845\u001e \u001fa2020-07-01\u001fb2021-02-01\u001e \u001faFacebook\u001e \u001fauser behavior tracking\u001e \u001faThis table is used to analyze the spread of different types of content on Facebook before and after the 2020 election, as part of the U.S. 2020 Facebook and Instagram Election Study. See the codebook for additional details.\u001e \u001faDownload the data dictionary for variables present in this dataset.\u001e \u001faThe data in this study were analyzed using R (version 4.4.1). The analysis code imports several R packages available on CRAN, including renv (1.0.11), dplyr (1.1.4), ggplot2 (3.5.1), xtable (1.8-4), and quantreg (5.98).\u001e \u001faR\u001e \u001faR 4.4.1\u001fdR packages available on CRAN, including renv (1.0.11), dplyr (1.1.4), ggplot2 (3.5.1), xtable (1.8-4), and quantreg (5.98)\u001e \u001fahttps://somar.infoready4.com/#freeformCompetitionDetail/1910437\u001e \u001faFacebook\u001e \u001faDatasets\u001e \u001faUS2020\u001e \u001faPublished\u001fb2024-12-10\u001e\u001d",
"items": [
{
"itemType": "book",
"title": "Diffusion Time Metrics for Facebook Posts with 100 or More Reshares",
"creators": [
{
"firstName": "Inc",
"lastName": "Meta Platforms",
"creatorType": "editor"
}
],
"abstractNote": "This dataset contains aggregated information about all content reshared 100 or more times from July 1, 2020 through February 1, 2021. Each row of the dataset corresponds to an individual tree and its size and depth at specific hours and days from initial posting",
"attachments": [],
"tags": [
{
"tag": "United States"
},
{
"tag": "elections"
},
{
"tag": "political attitudes"
},
{
"tag": "political behavior"
},
{
"tag": "social media"
},
{
"tag": "web platform data"
}
],
"notes": [
{
"note": "The U.S. 2020 Facebook and Instagram Election Study (US 2020 FIES) is a partnership between Meta and academic researchers to understand the impact of Facebook and Instagram on key political attitudes and behaviors during the US 2020 election"
}
],
"seeAlso": []
}
]
}
]
/** END TEST CASES **/
Loading
Loading