Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Transport layer tests #102

Open
wants to merge 8 commits into
base: master
Choose a base branch
from
Open
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions .mocharc.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,3 @@
module.exports = {
require: ["test/hooks.js"]
}
13 changes: 11 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
@@ -4,10 +4,19 @@ test:
./node_modules/.bin/mocha --recursive --timeout 10000 --exit ./test/unit ./test/integration

test-bail:
./node_modules/.bin/mocha --bail --recursive --timeout 10000 ./test/unit ./test/integration
./node_modules/.bin/mocha --bail --recursive --timeout 10000 --exit ./test/unit ./test/integration

test-integration:
./node_modules/.bin/mocha --recursive --timeout 10000 ./test/integration
./node_modules/.bin/mocha --recursive --timeout 10000 --exit ./test/integration

test-stats:
./node_modules/.bin/mocha --timeout 30000 ./test/integration/stats.js

test-data:
./node_modules/.bin/mocha --timeout 60000 --exit --bail ./test/integration/data.js

test-synergy:
./node_modules/.bin/mocha --timeout 10000 --exit --bail ./test/integration/synergy.js

test-unit:
./node_modules/.bin/mocha --recursive ./test/unit/
2 changes: 2 additions & 0 deletions package.json
Original file line number Diff line number Diff line change
@@ -18,6 +18,7 @@
"url": "https://github.com/bttmly/nba/issues"
},
"dependencies": {
"axios": "^0.21.1",
"camel-case": "^3.0.0",
"lodash.find": "^3.2.0",
"lodash.findwhere": "^3.1.0",
@@ -29,6 +30,7 @@
"minimist": "^1.2.0",
"nba-client-template": "4.5.0",
"node-fetch": "2.6.1",
"puppeteer": "^5.5.0",
"url": "^0.11.0"
},
"devDependencies": {
20 changes: 20 additions & 0 deletions scripts/run-transport.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
const { NBA_URL, TRANSPORT = "basic" } = process.env;
const { defaultTransport, setDefaultTransport } = require("../src/transport");
const { transport: puppeteerTransport } = require("../src/PuppeteerTransport");
const { URL } = require("url");
const transforms = require("../src/transforms");

if (!NBA_URL) throw new Error("must provide NBA_URL");

(async () => {
if (TRANSPORT !== "basic") {
setDefaultTransport(puppeteerTransport);
}


let url = new URL(NBA_URL).toString();
url = url.split("\\").join("");
const result = await defaultTransport(url);
console.log(transforms.general(result));
// console.log(JSON.stringify(result));
})();
138 changes: 138 additions & 0 deletions src/PuppeteerTransport.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,138 @@
const puppeteer = require("puppeteer");
const { URL } = require("url");

const USER_AGENT = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_0) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/78.0.3904.108 Safari/537.36";

const delay = (ms) => new Promise(r => setTimeout(r, ms));

class PuppeteerTransport {
static async create () {
const browser = await puppeteer.launch({
handleSIGINT: false,
handleSIGTERM: false,
args: [
"--disable-web-security",
],
});
return new PuppeteerTransport(browser);
}

constructor (browser) {
this.browser = browser;
}

async _createPage () {
const page = await this.browser.newPage();
await page.setRequestInterception(true);
page.on("request", req => {
const type = req.resourceType();
switch (type) {
case "document":
case "fetch":
req.continue();
break;
default:
req.abort();
}
});
page.on("framenavigated", (frame) => {
console.log("NAVIGATION:", frame.url(), "main", frame === page.mainFrame());
});
await page.setUserAgent(USER_AGENT);
await page.goto("https://www.nba.com/stats/", { waitUntil: "domcontentloaded", timeout: 8 * 1000 });
return page;
}

async _getPage () {
if (this.pageP) {
return this.pageP;
}
this.pageP = this._createPage();
return this.pageP;
}

async run (_url) {
const page = await this._getPage();
const result = await page.evaluate(async (url) => {
const headers = {
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en,en-US;q=0.9",
Accept: "application/json, text/plain, */*",
Referer: "https://www.nba.com/",
Connection: "keep-alive",
"Cache-Control": "no-cache",
Origin: "http://www.nba.com",
// "x-nba-stats-origin": "stats",
// "x-nba-stats-token": "true",
};

try {
const res = await fetch(url, { headers });
if (res.ok) {
const data = await res.json();
return { data, ok: true };
}
const text = await res.text();
return {
ok: false,
data: { text, status: res.status },
};
} catch (err) {
console.log(err);
return { ok: false, data: { text: err.toString() }};
}
}, _url);
return result;
}

close () {
console.trace("browser close");
return this.browser.close();
}
};

let instanceP = null;

async function transport (baseURL, query = {}) {
if (instanceP == null) {
instanceP = PuppeteerTransport.create();
await delay(1);
}
const instance = await instanceP;

const u = new URL(baseURL);
for (const [key, value] of Object.entries(query)) {
u.searchParams.append(key, value);
}
u.protocol = "https:";
const result = await instance.run(u.toString());
if (result.ok) return result.data;
throw new Error(`${result.data.text}${u.toString()}`);
}

module.exports.transport = transport;
module.exports.closeTransport = async () => {
if (instanceP == null) return;
const instance = await instanceP;
await instance.browser.close();
};
module.exports.PuppeteerTransport = PuppeteerTransport;

// (async () => {
// const urls = [
// "https://stats.nba.com/stats/leaguedashteamstats?Conference=&DateFrom=&DateTo=&Division=&GameScope=&GameSegment=&LastNGames=0&LeagueID=00&Location=&MeasureType=Advanced&Month=0&OpponentTeamID=0&Outcome=&PORound=0&PaceAdjust=N&PerMode=PerGame&Period=0&PlayerExperience=&PlayerPosition=&PlusMinus=N&Rank=N&Season=2020-21&SeasonSegment=&SeasonType=Regular+Season&ShotClockRange=&StarterBench=&TeamID=0&TwoWay=0&VsConference=&VsDivision=",
// "http://stats.nba.com/stats/playerprofilev2?LeagueID=00&PerMode=PerGame&PlayerID=201939&Season=2017-18",
// ];

// const b = await puppeteer.launch({
// args: [
// "--disable-web-security",
// ],
// });
// const p = new PuppeteerTransport(b);
// for (const u of urls) {
// const { ok, data } = await p.run(u);
// console.log(ok, data.text, u.split("?")[0]);
// }
// await p.close();
// })();
141 changes: 72 additions & 69 deletions src/data.js
Original file line number Diff line number Diff line change
@@ -1,78 +1,96 @@
// this includes endpoints at data.nba.com

let transport = require("./get-json");
const { defaultTransport } = require("./transport");
const { interpolate } = require("./util/string");

const scoreboardURL = interpolate("http://data.nba.com/data/5s/json/cms/noseason/scoreboard/__date__/games.json");
const boxScoreURL = interpolate("http://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/boxscore.json");
const playByPlayURL = interpolate("http://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/pbp_all.json");
const scheduleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/schedule.json");
const teamScheduleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/schedule.json");
const previewArticleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__date__/__gameId___preview_article.json");
const recapArticleURL = interpolate("http://data.nba.com/data/10s/prod/v1/__date__/__gameId___recap_article.json");
const leadTrackerURL = interpolate("http://data.nba.com/data/10s/prod/v1/__date__/__gameId___lead_tracker___period__.json");
const playoffsBracketURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/playoffsBracket.json");
const teamLeadersURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/leaders.json");
const teamStatsRankingsURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/team_stats_rankings.json");
const coachesURL = interpolate("http://data.nba.com/data/10s/prod/v1/__season__/coaches.json");
const teamsURL = interpolate("http://data.nba.net/data/10s/prod/v1/__year__/teams.json");

const calendarURL = "http://data.nba.net/data/10s/prod/v1/calendar.json";
const standingsURL = "http://data.nba.net/data/10s/prod/v1/current/standings_all.json";

const withTransport = (newTransport) => {
transport = newTransport;
};
const scoreboardURL = interpolate("https://data.nba.com/data/5s/json/cms/noseason/scoreboard/__date__/games.json");
const boxScoreURL = interpolate("https://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/boxscore.json");
const playByPlayURL = interpolate("https://data.nba.com/data/5s/json/cms/noseason/game/__date__/__gameId__/pbp_all.json");
const scheduleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/schedule.json");
const teamScheduleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/schedule.json");
const previewArticleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__date__/__gameId___preview_article.json");
const recapArticleURL = interpolate("https://data.nba.com/data/10s/prod/v1/__date__/__gameId___recap_article.json");
const leadTrackerURL = interpolate("https://data.nba.com/data/10s/prod/v1/__date__/__gameId___lead_tracker___period__.json");
const playoffsBracketURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/playoffsBracket.json");
const teamLeadersURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/teams/__teamId__/leaders.json");
const teamStatsRankingsURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/team_stats_rankings.json");
const coachesURL = interpolate("https://data.nba.com/data/10s/prod/v1/__season__/coaches.json");
const teamsURL = interpolate("https://data.nba.net/data/10s/prod/v1/__year__/teams.json");

const calendarURL = "https://data.nba.net/data/10s/prod/v1/calendar.json";
const standingsURL = "https://data.nba.net/data/10s/prod/v1/current/standings_all.json";

// NOTE: the 'date' argument should be a string in format like "20181008" (which indicates Oct 8 2018)
// You *can* pass a Date object but beware of timezone weirdness!

// NOTE: the 'season' argument is the first year of the NBA season e.g. "2018" for the 2018-19 season

const scoreboard = date => transport(scoreboardURL({ date: dateToYYYYMMDD(date) }));
scoreboard.defaults = { date: null };
const withTransport = (transportOverride) => {
const transport = transportOverride || defaultTransport;

const scoreboard = date => transport(scoreboardURL({ date: dateToYYYYMMDD(date) }));
scoreboard.defaults = { date: null };

const boxScore = (date, gameId) => transport(boxScoreURL({ date: dateToYYYYMMDD(date), gameId }));
boxScore.defaults = { date: null, gameId: null };
const boxScore = (date, gameId) => transport(boxScoreURL({ date: dateToYYYYMMDD(date), gameId }));
boxScore.defaults = { date: null, gameId: null };

const playByPlay = (date, gameId) => transport(playByPlayURL({ date: dateToYYYYMMDD(date), gameId }));
playByPlay.defaults = { date: null, gameId: null };
const playByPlay = (date, gameId) => transport(playByPlayURL({ date: dateToYYYYMMDD(date), gameId }));
playByPlay.defaults = { date: null, gameId: null };

const schedule = (season) => transport(scheduleURL({ season }));
schedule.defaults = { season: null };
const schedule = (season) => transport(scheduleURL({ season }));
schedule.defaults = { season: null };

const teamSchedule = (season, teamId) => transport(teamScheduleURL({ season, teamId }));
teamSchedule.defaults = { season: null, teamId: null };
const teamSchedule = (season, teamId) => transport(teamScheduleURL({ season, teamId }));
teamSchedule.defaults = { season: null, teamId: null };

const previewArticle = (date, gameId) => transport(previewArticleURL({date: dateToYYYYMMDD(date), gameId }));
previewArticle.defaults = { date: null, gameId: null };
const previewArticle = (date, gameId) => transport(previewArticleURL({date: dateToYYYYMMDD(date), gameId }));
previewArticle.defaults = { date: null, gameId: null };

const recapArticle = (date, gameId) => transport(recapArticleURL({date: dateToYYYYMMDD(date), gameId }));
recapArticle.defaults = { date: null, gameId: null };
const recapArticle = (date, gameId) => transport(recapArticleURL({date: dateToYYYYMMDD(date), gameId }));
recapArticle.defaults = { date: null, gameId: null };

const leadTracker = (date, gameId, period) => transport(leadTrackerURL({date: dateToYYYYMMDD(date), gameId, period }));
leadTracker.defaults = { date: null, gameId: null, period: null };
const leadTracker = (date, gameId, period) => transport(leadTrackerURL({date: dateToYYYYMMDD(date), gameId, period }));
leadTracker.defaults = { date: null, gameId: null, period: null };

const playoffsBracket = (season) => transport(playoffsBracketURL({ season }));
playoffsBracket.defaults = { season: null };
const playoffsBracket = (season) => transport(playoffsBracketURL({ season }));
playoffsBracket.defaults = { season: null };

const teamLeaders = (season, teamId) => transport(teamLeadersURL({ season, teamId }));
teamLeaders.defaults = { season: null, teamId: null };
const teamLeaders = (season, teamId) => transport(teamLeadersURL({ season, teamId }));
teamLeaders.defaults = { season: null, teamId: null };

const teamStatsRankings = (season) => transport(teamStatsRankingsURL({ season }));
teamStatsRankings.defaults = { season: null };
const teamStatsRankings = (season) => transport(teamStatsRankingsURL({ season }));
teamStatsRankings.defaults = { season: null };

const coaches = (season) => transport(coachesURL({ season }));
coaches.defaults = { season: null };
const coaches = (season) => transport(coachesURL({ season }));
coaches.defaults = { season: null };

const teams = (year = "2019") => transport(teamsURL({ year }));
teams.defaults = { year: null };
const teams = (year = "2019") => transport(teamsURL({ year }));
teams.defaults = { year: null };

const calendar = () => transport(calendarURL);
calendar.defaults = {};
const calendar = () => transport(calendarURL);
calendar.defaults = {};

const standings = () => transport(standingsURL);
standings.defaults = {};
const standings = () => transport(standingsURL);
standings.defaults = {};

return {
scoreboard,
boxScore,
playByPlay,
schedule,
teamSchedule,
previewArticle,
recapArticle,
leadTracker,
playoffsBracket,
teamLeaders,
teamStatsRankings,
coaches,
teams,
calendar,
standings,
};
};

function dateToYYYYMMDD (date) {
if (date instanceof Date) {
@@ -82,27 +100,12 @@ function dateToYYYYMMDD (date) {
String(date.getDate()).padStart(2, 0),
].join("");
}

// TODO: better checking here?

return date;
}

const client = withTransport();

module.exports = {
scoreboard,
boxScore,
playByPlay,
schedule,
teamSchedule,
previewArticle,
recapArticle,
leadTracker,
playoffsBracket,
teamLeaders,
teamStatsRankings,
coaches,
teams,
calendar,
standings,
...client,
withTransport,
};
39 changes: 20 additions & 19 deletions src/get-json.js
Original file line number Diff line number Diff line change
@@ -1,16 +1,16 @@
const url = require("url");
const template = require("nba-client-template");
const fetch = require("node-fetch");
const axios = require("axios");

const HEADERS = {
"Accept-Encoding": "gzip, deflate",
"Accept-Language": "en-US",
Accept: "*/*",
"Accept-Encoding": "gzip, deflate, br",
"Accept-Language": "en,en-US;q=0.9",
Accept: "application/json, text/plain, */*",
"User-Agent": template.user_agent,
Referer: template.referrer,
Referer: "www.nba.com",
Connection: "keep-alive",
"Cache-Control": "no-cache",
Origin: "http://stats.nba.com",
Origin: "https://www.nba.com",
};

function createUrlString (_url, query) {
@@ -19,22 +19,23 @@ function createUrlString (_url, query) {
return urlObj.format();
}

module.exports = function getJson (_url, query, _options = {}) {
const urlStr = createUrlString(_url, query);

async function getJsonAxios (_url, params, _options = {}) {
const options = {
..._options,
headers: { ..._options.headers, ...HEADERS },
headers: { ...HEADERS, ..._options.headers },
};

return fetch(urlStr, options)
.then(resp => {
if (resp.ok) return resp.json();

return resp.text().then(function (text) {
throw new Error(`${resp.status} ${resp.statusText}${text}`);
});
});
const urlStr = createUrlString(_url, params);
try {
const res = await axios.get(urlStr, options);
return res.data;
} catch (err) {
const { response } = err;
if (response) {
throw new Error(`${response.status} ${response.data}`);
}
throw new Error(`Unknown HTTP error for ${url}`);
}
};


module.exports = getJsonAxios;
14 changes: 9 additions & 5 deletions src/stats.js
Original file line number Diff line number Diff line change
@@ -46,13 +46,14 @@ function makeStatsMethod (endpoint, transport) {
const ccName = camelCase(endpoint.name);
const transform = transformMap[ccName];

function statsMethod (query = {}) {
const reqParams = Object.assign({}, defaults, query);
async function statsMethod (query = {}) {
const reqParams = { ...defaults, ...query };

const options = {
headers: {
"x-nba-stats-origin": "stats",
"x-nba-stats-token": "true",
// "x-nba-stats-origin": "stats",
// "x-nba-stats-token": "true",
// "user-agent": "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/88.0.4324.96 Safari/537.36",
},
};

@@ -80,4 +81,7 @@ function makeStatsClient (transport) {
return client;
}

module.exports = makeStatsClient(require("./get-json"));
const { defaultTransport } = require("./transport");

module.exports = makeStatsClient(defaultTransport);

8 changes: 5 additions & 3 deletions src/synergy.js
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
const camelCase = require("camel-case");
const { defaultTransport } = require("./transport");

const parameters = [
{
@@ -44,8 +45,8 @@ const parameters = [
];

const synergyEndpoints = [
{ name: "player_play_type", url: "http://stats-prod.nba.com/wp-json/statscms/v1/synergy/player/" },
{ name: "team_play_type", url: "http://stats-prod.nba.com/wp-json/statscms/v1/synergy/team/" },
{ name: "player_play_type", url: "https://stats-prod.nba.com/wp-json/statscms/v1/synergy/player/" },
{ name: "team_play_type", url: "https://stats-prod.nba.com/wp-json/statscms/v1/synergy/team/" },
];

const defaults = {};
@@ -76,4 +77,5 @@ function makeSynergyClient (transport) {
return client;
}

module.exports = makeSynergyClient(require("./get-json"));

module.exports = makeSynergyClient(defaultTransport);
11 changes: 11 additions & 0 deletions src/transport.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
let _defaultTransport = require("./get-json");

function defaultTransport (...args) {
return _defaultTransport(...args);
}

function setDefaultTransport (t) {
_defaultTransport = t;
}

module.exports = { defaultTransport, setDefaultTransport };
18 changes: 0 additions & 18 deletions test/get-json-stub.js

This file was deleted.

11 changes: 11 additions & 0 deletions test/hooks.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,11 @@
const { setDefaultTransport } = require("../src/transport");
const p = require("../src/PuppeteerTransport");

exports.mochaHooks = {
beforeAll () {
setDefaultTransport(p.transport);
},
async afterAll () {
await p.closeTransport();
},
};
17 changes: 0 additions & 17 deletions test/inspect.js

This file was deleted.

2 changes: 1 addition & 1 deletion test/integration/data.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
const nba = require("../../");

describe("nba data methods", function () {
describe("#scoreboard", async () => {
describe("#scoreboard", () => {
it("works with a direct date string", async () => {
log(await nba.data.scoreboard("20181008"));
});
25 changes: 11 additions & 14 deletions test/integration/stats.js
Original file line number Diff line number Diff line change
@@ -6,8 +6,6 @@ const pify = require("pify");

const nba = require("../../");

// for interactive inspection, particularly in browser
global.StatsData = {};
const tested = {};
const methods = {};

@@ -33,7 +31,7 @@ const callMethod = (name, params = {}, shape) => async () => {
params.Season = "2017-18";
const r = await stats[name](params);
verifyShape(shape, r);
global.StatsData[name] = r;
await writeResponse(name, r);
};

const _steph = 201939;
@@ -81,19 +79,18 @@ describe("nba stats methods", function () {
it("#leagueStandings", callMethod("leagueStandings"));
it("#teamPlayerOnOffDetails", callMethod("teamPlayerOnOffDetails", { TeamID: _dubs }));
it("#playerCompare", callMethod("playerCompare", { PlayerIDList: _steph, VsPlayerIDList: _steph }));


after(function () {
return Promise.all(Object.keys(global.StatsData).map(k =>
pify(fs.writeFile)(
path.join(__dirname, "../../responses", `stats-${k}.json`),
JSON.stringify(global.StatsData[k], null, 2)
)
))
.catch(console.error);
});
});

const writeFile = pify(fs.writeFile);
async function writeResponse (key, value) {
try {
await writeFile(
path.join(__dirname, "../../responses", `stats-${key}.json`),
JSON.stringify(value, null, 2)
);
} catch (e) {}
}

// describe("tested all methods", function () {
// it("did test all methods", () => {
// try {
6 changes: 3 additions & 3 deletions test/integration/synergy.js
Original file line number Diff line number Diff line change
@@ -7,15 +7,15 @@ const writeFile = pify(require("fs").writeFile);
const dir = path.join(__dirname, "../../responses");
function writeData (name, data) {
const str = JSON.stringify(data, null, 2);
return writeFile(path.join(dir, `synergy-${name}.json`), str);
return writeFile(path.join(dir, `synergy-${name}.json`), str).catch(() => {});
}

global.SynergyData = {};

// stub for now, will add response shape verification for self-documenting responses
const verifyShape = shape => response => response;
// const verifyShape = shape => response => response;

const callMethod = (name, params = {}, shape) => () => {
const callMethod = (name, params = {}) => () => {
params.season = 2016;
return nba.synergy[name](params)
.then(function (resp) {
22 changes: 0 additions & 22 deletions test/nba-api-spy.js

This file was deleted.

31 changes: 31 additions & 0 deletions test/scratchpad.js
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
const { getJsonNodeFetch, getJsonAxios } = require("../src/get-json");

async function main () {
const url = "https://stats.nba.com/stats/leagueLeaders";
const query = {
"ActiveFlag": "No",
"LeagueID": "00",
"PerMode": "Totals",
"Scope": "S",
"Season": "All Time",
"SeasonType": "Regular Season",
"StatCategory": "PTS",
};
let err, result;

({ err, result } = await settle(getJsonNodeFetch(url, query)));
console.log({ err, result });
({ err, result } = await settle(getJsonAxios(url, query)));
console.log({ err, result });
}

async function settle (p) {
try {
const result = await p;
return { result };
} catch (err) {
return { err };
}
}

main();
293 changes: 280 additions & 13 deletions yarn.lock

Large diffs are not rendered by default.