exquisitecorp
/
ArtArchive
mirror of https://github.com/blprnt/ArtArchive.git
1
0
Fork 0
Code Issues Packages Projects Releases Wiki Activity

Adding class 2 files.

This commit is contained in:
Jer Thorp 2019-02-06 10:24:07 -05:00
parent 709a3f17e7
commit e2d27fe555
8 changed files with 1109 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
Class2/marc_template/.gitignore vendored Normal file
View File

@ -0,0 +1,2 @@
data/*.gz
node_modules/*

9
Class2/marc_template/README.md Normal file
View File

@ -0,0 +1,9 @@
# Experiments with MARC parsing (node.js)
To run:
- install node.js
- in terminal:
- npm install
- npm download-data
- npm start

1
Class2/marc_template/data/urls.txt Normal file
View File

@ -0,0 +1 @@
http://www.loc.gov/cds/downloads/MDSConnect/Visual.Materials.2014.part01.xml.gz

42
Class2/marc_template/data/urls_books.txt Normal file
View File

@ -0,0 +1,42 @@
http://www.loc.gov/cds/downloads/MDSConnect/Visual.Materials.2014.part01.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part01.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part02.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part03.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part04.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part05.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part06.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part07.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part08.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part09.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part10.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part11.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part12.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part13.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part14.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part15.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part16.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part17.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part18.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part19.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part20.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part21.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part22.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part23.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part24.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part25.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part26.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part27.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part28.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part29.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part30.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part31.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part32.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part33.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part34.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part35.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part36.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part37.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part38.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part39.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part40.xml.gz
http://www.loc.gov/cds/downloads/MDSConnect/BooksAll.2014.part41.xml.gz

152
Class2/marc_template/index.js Normal file
View File

@ -0,0 +1,152 @@
/*
Node.js parser for MARC Files from Library of Congress
- Make network JSON files for consumption by sigma.js
- npm run download-data to get data files (you may have to install wget)
Jer Thorp (@blprnt)
December, 2017
*/
let request = require('request');
const fs = require('fs');
const zlib = require('zlib');
const concat = require('concat-stream');
const xml2object = require('xml2object');
const appRoot = require('app-root-path');
const natural = require('natural');
var dataPath = appRoot + "/data";
const marc_location = dataPath;
const filePrefix = "Visual.Materials"
var docCount = 0;
var docCounts = [];
var callNumCounts = [];
//XML Parser
var parser;
// Create a new xml parser looking for the record objects
function makeParser() {
parser = new xml2object([ 'record' ]);
parser.outs = [];
parser.on('object', function(name, obj) {
parseRecord(obj);
});
parser.on('end', function() {
console.log('Finished parsing xml!');
onParseFinished();
});
}
//Record parser
//Parse MARC record into a usable JSON object
//https://folgerpedia.folger.edu/Interpreting_MARC_records#2xx
//SUPER rough for now!
const marcDict = {};
marcDict["245"] = {"a" :"Title"};
var nameDict = {};
var lastNameDict = {};
var callDict = {};
var callYearTotals = [];
var allRecords = [];
function parseRecord(obj) {
record = {};
for (var i = 0; i < obj.datafield.length; i++) {
var df = obj.datafield[i];
//Get the numeric tag
var tag = df.tag;
//If we have the tag in our dictionary, write to the JSON object
//Based on the code (doesn't work for all cases?)
if (marcDict[tag]) {
for (var j = 0; j < df.subfield.length; j++) {
var code = df.subfield[j].code;
var disp = df.subfield[j]['$t'];
if (marcDict[tag][code]) {
if (!record[marcDict[tag][code]]) {
record[marcDict[tag][code]] = [];
}
record[marcDict[tag][code]].push(disp);
}
}
}
}
if (record.Title) {
if (record.Title.length > 0) {
var t = record.Title[0];
var chk = checkForColor(t);
if (chk.chk) console.log(chk.col + ":" + record.Title[0]);
}
}
}
function checkForColor(_s) {
var colors = [" red"," blue"," green"];
var chk = {chk:false, col:null};
for (var i = 0; i < colors.length; i++) {
if (_s.indexOf(colors[i]) != -1) {
chk.chk = true;
chk.col = colors[i];
}
}
return(chk);
}
function incrementDict(dict, val, yi) {
if (!dict[val]) {
dict[val] = {
"name":val,
"total":0,
"years":[],
"callNums":{}
};
}
}
function onParseFinished() {
nextFile();
}
var counter = 1;
function nextFile() {
var n = (counter < 10 ? "0":"") + counter;
var url = marc_location + "/" + filePrefix + ".2014.part" + n + ".xml.gz";
var rstream = fs.createReadStream(url);
var gunzip = zlib.createGunzip();
makeParser();
allRecords = [];
console.log("LOADING FILE : " + url);
counter ++;
if (counter < 42) {
rstream // reads from myfile.txt.gz
.pipe(gunzip) // uncompresses
.pipe(parser.saxStream); //Parses into record objects
}
}
nextFile();

863
Class2/marc_template/package-lock.json generated Normal file
View File

@ -0,0 +1,863 @@
{
"name": "LOC_MARC",
"version": "1.0.0",
"lockfileVersion": 1,
"requires": true,
"dependencies": {
"accepts": {
"version": "1.3.4",
"resolved": "https://registry.npmjs.org/accepts/-/accepts-1.3.4.tgz",
"integrity": "sha1-hiRnWMfdbSGmR0/whKR0DsBesh8=",
"requires": {
"mime-types": "2.1.17",
"negotiator": "0.6.1"
}
},
"ajv": {
"version": "5.5.2",
"resolved": "https://registry.npmjs.org/ajv/-/ajv-5.5.2.tgz",
"integrity": "sha1-c7Xuyj+rZT49P5Qis0GtQiBdyWU=",
"requires": {
"co": "4.6.0",
"fast-deep-equal": "1.0.0",
"fast-json-stable-stringify": "2.0.0",
"json-schema-traverse": "0.3.1"
}
},
"app-root-path": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/app-root-path/-/app-root-path-2.0.1.tgz",
"integrity": "sha1-zWLc+OT9WkF+/GZNLlsQZTxlG0Y="
},
"apparatus": {
"version": "0.0.9",
"resolved": "https://registry.npmjs.org/apparatus/-/apparatus-0.0.9.tgz",
"integrity": "sha1-N9zSWDStC2UQdllikduCPusZCL0=",
"requires": {
"sylvester": "0.0.21"
}
},
"array-flatten": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/array-flatten/-/array-flatten-1.1.1.tgz",
"integrity": "sha1-ml9pkFGx5wczKPKgCJaLZOopVdI="
},
"asn1": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/asn1/-/asn1-0.2.3.tgz",
"integrity": "sha1-2sh4dxPJlmhJ/IGAd36+nB3fO4Y="
},
"assert-plus": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/assert-plus/-/assert-plus-1.0.0.tgz",
"integrity": "sha1-8S4PPF13sLHN2RRpQuTpbB5N1SU="
},
"asynckit": {
"version": "0.4.0",
"resolved": "https://registry.npmjs.org/asynckit/-/asynckit-0.4.0.tgz",
"integrity": "sha1-x57Zf380y48robyXkLzDZkdLS3k="
},
"aws-sign2": {
"version": "0.7.0",
"resolved": "https://registry.npmjs.org/aws-sign2/-/aws-sign2-0.7.0.tgz",
"integrity": "sha1-tG6JCTSpWR8tL2+G1+ap8bP+dqg="
},
"aws4": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/aws4/-/aws4-1.6.0.tgz",
"integrity": "sha1-g+9cqGCysy5KDe7e6MdxudtXRx4="
},
"bcrypt-pbkdf": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/bcrypt-pbkdf/-/bcrypt-pbkdf-1.0.1.tgz",
"integrity": "sha1-Y7xdy2EzG5K8Bf1SiVPDNGKgb40=",
"optional": true,
"requires": {
"tweetnacl": "0.14.5"
}
},
"bindings": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/bindings/-/bindings-1.3.0.tgz",
"integrity": "sha512-DpLh5EzMR2kzvX1KIlVC0VkC3iZtHKTgdtZ0a3pglBZdaQFjt5S9g9xd1lE+YvXyfd6mtCeRnrUfOLYiTMlNSw==",
"optional": true
},
"body-parser": {
"version": "1.18.2",
"resolved": "https://registry.npmjs.org/body-parser/-/body-parser-1.18.2.tgz",
"integrity": "sha1-h2eKGdhLR9hZuDGZvVm84iKxBFQ=",
"requires": {
"bytes": "3.0.0",
"content-type": "1.0.4",
"debug": "2.6.9",
"depd": "1.1.2",
"http-errors": "1.6.2",
"iconv-lite": "0.4.19",
"on-finished": "2.3.0",
"qs": "6.5.1",
"raw-body": "2.3.2",
"type-is": "1.6.15"
}
},
"boom": {
"version": "4.3.1",
"resolved": "https://registry.npmjs.org/boom/-/boom-4.3.1.tgz",
"integrity": "sha1-T4owBctKfjiJ90kDD9JbluAdLjE=",
"requires": {
"hoek": "4.2.0"
}
},
"bytes": {
"version": "3.0.0",
"resolved": "https://registry.npmjs.org/bytes/-/bytes-3.0.0.tgz",
"integrity": "sha1-0ygVQE1olpn4Wk6k+odV3ROpYEg="
},
"caseless": {
"version": "0.12.0",
"resolved": "https://registry.npmjs.org/caseless/-/caseless-0.12.0.tgz",
"integrity": "sha1-G2gcIf+EAzyCZUMJBolCDRhxUdw="
},
"co": {
"version": "4.6.0",
"resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
"integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
},
"combined-stream": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/combined-stream/-/combined-stream-1.0.5.tgz",
"integrity": "sha1-k4NwpXtKUd6ix3wV1cX9+JUWQAk=",
"requires": {
"delayed-stream": "1.0.0"
}
},
"concat-stream": {
"version": "1.6.0",
"resolved": "https://registry.npmjs.org/concat-stream/-/concat-stream-1.6.0.tgz",
"integrity": "sha1-CqxmL9Ur54lk1VMvaUeE5wEQrPc=",
"requires": {
"inherits": "2.0.3",
"readable-stream": "2.3.3",
"typedarray": "0.0.6"
}
},
"content-disposition": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/content-disposition/-/content-disposition-0.5.2.tgz",
"integrity": "sha1-DPaLud318r55YcOoUXjLhdunjLQ="
},
"content-type": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/content-type/-/content-type-1.0.4.tgz",
"integrity": "sha512-hIP3EEPs8tB9AT1L+NUqtwOAps4mk2Zob89MWXMHjHWg9milF/j4osnnQLXBCBFBk/tvIG/tUc9mOUJiPBhPXA=="
},
"cookie": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/cookie/-/cookie-0.3.1.tgz",
"integrity": "sha1-5+Ch+e9DtMi6klxcWpboBtFoc7s="
},
"cookie-signature": {
"version": "1.0.6",
"resolved": "https://registry.npmjs.org/cookie-signature/-/cookie-signature-1.0.6.tgz",
"integrity": "sha1-4wOogrNCzD7oylE6eZmXNNqzriw="
},
"core-util-is": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz",
"integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac="
},
"cryptiles": {
"version": "3.1.2",
"resolved": "https://registry.npmjs.org/cryptiles/-/cryptiles-3.1.2.tgz",
"integrity": "sha1-qJ+7Ig9c4l7FboxKqKT9e1sNKf4=",
"requires": {
"boom": "5.2.0"
},
"dependencies": {
"boom": {
"version": "5.2.0",
"resolved": "https://registry.npmjs.org/boom/-/boom-5.2.0.tgz",
"integrity": "sha512-Z5BTk6ZRe4tXXQlkqftmsAUANpXmuwlsF5Oov8ThoMbQRzdGTA1ngYRW160GexgOgjsFOKJz0LYhoNi+2AMBUw==",
"requires": {
"hoek": "4.2.0"
}
}
}
},
"dashdash": {
"version": "1.14.1",
"resolved": "https://registry.npmjs.org/dashdash/-/dashdash-1.14.1.tgz",
"integrity": "sha1-hTz6D3y+L+1d4gMmuN1YEDX24vA=",
"requires": {
"assert-plus": "1.0.0"
}
},
"debug": {
"version": "2.6.9",
"resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz",
"integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==",
"requires": {
"ms": "2.0.0"
}
},
"delayed-stream": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/delayed-stream/-/delayed-stream-1.0.0.tgz",
"integrity": "sha1-3zrhmayt+31ECqrgsp4icrJOxhk="
},
"depd": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.2.tgz",
"integrity": "sha1-m81S4UwJd2PnSbJ0xDRu0uVgtak="
},
"destroy": {
"version": "1.0.4",
"resolved": "https://registry.npmjs.org/destroy/-/destroy-1.0.4.tgz",
"integrity": "sha1-l4hXRCxEdJ5CBmE+N5RiBYJqvYA="
},
"ecc-jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/ecc-jsbn/-/ecc-jsbn-0.1.1.tgz",
"integrity": "sha1-D8c6ntXw1Tw4GTOYUj735UN3dQU=",
"optional": true,
"requires": {
"jsbn": "0.1.1"
}
},
"ee-first": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/ee-first/-/ee-first-1.1.1.tgz",
"integrity": "sha1-WQxhFWsK4vTwJVcyoViyZrxWsh0="
},
"en-stemmer": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/en-stemmer/-/en-stemmer-1.0.3.tgz",
"integrity": "sha1-LSL7XbkAWojW/FgCazioMci8fBk="
},
"encodeurl": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/encodeurl/-/encodeurl-1.0.2.tgz",
"integrity": "sha1-rT/0yG7C0CkyL1oCw6mmBslbP1k="
},
"escape-html": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/escape-html/-/escape-html-1.0.3.tgz",
"integrity": "sha1-Aljq5NPQwJdN4cFpGI7wBR0dGYg="
},
"etag": {
"version": "1.8.1",
"resolved": "https://registry.npmjs.org/etag/-/etag-1.8.1.tgz",
"integrity": "sha1-Qa4u62XvpiJorr/qg6x9eSmbCIc="
},
"express": {
"version": "4.16.2",
"resolved": "https://registry.npmjs.org/express/-/express-4.16.2.tgz",
"integrity": "sha1-41xt/i1kt9ygpc1PIXgb4ymeB2w=",
"requires": {
"accepts": "1.3.4",
"array-flatten": "1.1.1",
"body-parser": "1.18.2",
"content-disposition": "0.5.2",
"content-type": "1.0.4",
"cookie": "0.3.1",
"cookie-signature": "1.0.6",
"debug": "2.6.9",
"depd": "1.1.2",
"encodeurl": "1.0.2",
"escape-html": "1.0.3",
"etag": "1.8.1",
"finalhandler": "1.1.0",
"fresh": "0.5.2",
"merge-descriptors": "1.0.1",
"methods": "1.1.2",
"on-finished": "2.3.0",
"parseurl": "1.3.2",
"path-to-regexp": "0.1.7",
"proxy-addr": "2.0.2",
"qs": "6.5.1",
"range-parser": "1.2.0",
"safe-buffer": "5.1.1",
"send": "0.16.1",
"serve-static": "1.13.1",
"setprototypeof": "1.1.0",
"statuses": "1.3.1",
"type-is": "1.6.15",
"utils-merge": "1.0.1",
"vary": "1.1.2"
}
},
"extend": {
"version": "3.0.1",
"resolved": "https://registry.npmjs.org/extend/-/extend-3.0.1.tgz",
"integrity": "sha1-p1Xqe8Gt/MWjHOfnYtuq3F5jZEQ="
},
"extsprintf": {
"version": "1.3.0",
"resolved": "https://registry.npmjs.org/extsprintf/-/extsprintf-1.3.0.tgz",
"integrity": "sha1-lpGEQOMEGnpBT4xS48V06zw+HgU="
},
"fast-deep-equal": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/fast-deep-equal/-/fast-deep-equal-1.0.0.tgz",
"integrity": "sha1-liVqO8l1WV6zbYLpkp0GDYk0Of8="
},
"fast-json-stable-stringify": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/fast-json-stable-stringify/-/fast-json-stable-stringify-2.0.0.tgz",
"integrity": "sha1-1RQsDK7msRifh9OnYREGT4bIu/I="
},
"finalhandler": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/finalhandler/-/finalhandler-1.1.0.tgz",
"integrity": "sha1-zgtoVbRYU+eRsvzGgARtiCU91/U=",
"requires": {
"debug": "2.6.9",
"encodeurl": "1.0.2",
"escape-html": "1.0.3",
"on-finished": "2.3.0",
"parseurl": "1.3.2",
"statuses": "1.3.1",
"unpipe": "1.0.0"
}
},
"forever-agent": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/forever-agent/-/forever-agent-0.6.1.tgz",
"integrity": "sha1-+8cfDEGt6zf5bFd60e1C2P2sypE="
},
"form-data": {
"version": "2.3.1",
"resolved": "https://registry.npmjs.org/form-data/-/form-data-2.3.1.tgz",
"integrity": "sha1-b7lPvXGIUwbXPRXMSX/kzE7NRL8=",
"requires": {
"asynckit": "0.4.0",
"combined-stream": "1.0.5",
"mime-types": "2.1.17"
}
},
"forwarded": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/forwarded/-/forwarded-0.1.2.tgz",
"integrity": "sha1-mMI9qxF1ZXuMBXPozszZGw/xjIQ="
},
"fresh": {
"version": "0.5.2",
"resolved": "https://registry.npmjs.org/fresh/-/fresh-0.5.2.tgz",
"integrity": "sha1-PYyt2Q2XZWn6g1qx+OSyOhBWBac="
},
"fs": {
"version": "0.0.1-security",
"resolved": "https://registry.npmjs.org/fs/-/fs-0.0.1-security.tgz",
"integrity": "sha1-invTcYa23d84E/I4WLV+yq9eQdQ="
},
"getpass": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/getpass/-/getpass-0.1.7.tgz",
"integrity": "sha1-Xv+OPmhNVprkyysSgmBOi6YhSfo=",
"requires": {
"assert-plus": "1.0.0"
}
},
"har-schema": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/har-schema/-/har-schema-2.0.0.tgz",
"integrity": "sha1-qUwiJOvKwEeCoNkDVSHyRzW37JI="
},
"har-validator": {
"version": "5.0.3",
"resolved": "https://registry.npmjs.org/har-validator/-/har-validator-5.0.3.tgz",
"integrity": "sha1-ukAsJmGU8VlW7xXg/PJCmT9qff0=",
"requires": {
"ajv": "5.5.2",
"har-schema": "2.0.0"
}
},
"hawk": {
"version": "6.0.2",
"resolved": "https://registry.npmjs.org/hawk/-/hawk-6.0.2.tgz",
"integrity": "sha512-miowhl2+U7Qle4vdLqDdPt9m09K6yZhkLDTWGoUiUzrQCn+mHHSmfJgAyGaLRZbPmTqfFFjRV1QWCW0VWUJBbQ==",
"requires": {
"boom": "4.3.1",
"cryptiles": "3.1.2",
"hoek": "4.2.0",
"sntp": "2.1.0"
}
},
"hoek": {
"version": "4.2.0",
"resolved": "https://registry.npmjs.org/hoek/-/hoek-4.2.0.tgz",
"integrity": "sha512-v0XCLxICi9nPfYrS9RL8HbYnXi9obYAeLbSP00BmnZwCK9+Ih9WOjoZ8YoHCoav2csqn4FOz4Orldsy2dmDwmQ=="
},
"http-errors": {
"version": "1.6.2",
"resolved": "https://registry.npmjs.org/http-errors/-/http-errors-1.6.2.tgz",
"integrity": "sha1-CgAsyFcHGSp+eUbO7cERVfYOxzY=",
"requires": {
"depd": "1.1.1",
"inherits": "2.0.3",
"setprototypeof": "1.0.3",
"statuses": "1.3.1"
},
"dependencies": {
"depd": {
"version": "1.1.1",
"resolved": "https://registry.npmjs.org/depd/-/depd-1.1.1.tgz",
"integrity": "sha1-V4O04cRZ8G+lyif5kfPQbnoxA1k="
},
"setprototypeof": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.0.3.tgz",
"integrity": "sha1-ZlZ+NwQ+608E2RvWWMDL77VbjgQ="
}
}
},
"http-signature": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/http-signature/-/http-signature-1.2.0.tgz",
"integrity": "sha1-muzZJRFHcvPZW2WmCruPfBj7rOE=",
"requires": {
"assert-plus": "1.0.0",
"jsprim": "1.4.1",
"sshpk": "1.13.1"
}
},
"iconv-lite": {
"version": "0.4.19",
"resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.19.tgz",
"integrity": "sha512-oTZqweIP51xaGPI4uPa56/Pri/480R+mo7SeU+YETByQNhDG55ycFyNLIgta9vXhILrxXDmF7ZGhqZIcuN0gJQ=="
},
"inherits": {
"version": "2.0.3",
"resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.3.tgz",
"integrity": "sha1-Yzwsg+PaQqUC9SRmAiSA9CCCYd4="
},
"ipaddr.js": {
"version": "1.5.2",
"resolved": "https://registry.npmjs.org/ipaddr.js/-/ipaddr.js-1.5.2.tgz",
"integrity": "sha1-1LUFvemUaYfM8PxY2QEP+WB+P6A="
},
"is-typedarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/is-typedarray/-/is-typedarray-1.0.0.tgz",
"integrity": "sha1-5HnICFjfDBsR3dppQPlgEfzaSpo="
},
"isarray": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz",
"integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE="
},
"isstream": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/isstream/-/isstream-0.1.2.tgz",
"integrity": "sha1-R+Y/evVa+m+S4VAOaQ64uFKcCZo="
},
"jsbn": {
"version": "0.1.1",
"resolved": "https://registry.npmjs.org/jsbn/-/jsbn-0.1.1.tgz",
"integrity": "sha1-peZUwuWi3rXyAdls77yoDA7y9RM=",
"optional": true
},
"json-schema": {
"version": "0.2.3",
"resolved": "https://registry.npmjs.org/json-schema/-/json-schema-0.2.3.tgz",
"integrity": "sha1-tIDIkuWaLwWVTOcnvT8qTogvnhM="
},
"json-schema-traverse": {
"version": "0.3.1",
"resolved": "https://registry.npmjs.org/json-schema-traverse/-/json-schema-traverse-0.3.1.tgz",
"integrity": "sha1-NJptRMU6Ud6JtAgFxdXlm0F9M0A="
},
"json-stringify-safe": {
"version": "5.0.1",
"resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz",
"integrity": "sha1-Epai1Y/UXxmg9s4B1lcB4sc1tus="
},
"jsprim": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/jsprim/-/jsprim-1.4.1.tgz",
"integrity": "sha1-MT5mvB5cwG5Di8G3SZwuXFastqI=",
"requires": {
"assert-plus": "1.0.0",
"extsprintf": "1.3.0",
"json-schema": "0.2.3",
"verror": "1.10.0"
}
},
"media-typer": {
"version": "0.3.0",
"resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz",
"integrity": "sha1-hxDXrwqmJvj/+hzgAWhUUmMlV0g="
},
"merge-descriptors": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/merge-descriptors/-/merge-descriptors-1.0.1.tgz",
"integrity": "sha1-sAqqVW3YtEVoFQ7J0blT8/kMu2E="
},
"methods": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/methods/-/methods-1.1.2.tgz",
"integrity": "sha1-VSmk1nZUE07cxSZmVoNbD4Ua/O4="
},
"mime": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/mime/-/mime-1.4.1.tgz",
"integrity": "sha512-KI1+qOZu5DcW6wayYHSzR/tXKCDC5Om4s1z2QJjDULzLcmf3DvzS7oluY4HCTrc+9FiKmWUgeNLg7W3uIQvxtQ=="
},
"mime-db": {
"version": "1.30.0",
"resolved": "https://registry.npmjs.org/mime-db/-/mime-db-1.30.0.tgz",
"integrity": "sha1-dMZD2i3Z1qRTmZY0ZbJtXKfXHwE="
},
"mime-types": {
"version": "2.1.17",
"resolved": "https://registry.npmjs.org/mime-types/-/mime-types-2.1.17.tgz",
"integrity": "sha1-Cdejk/A+mVp5+K+Fe3Cp4KsWVXo=",
"requires": {
"mime-db": "1.30.0"
}
},
"ms": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz",
"integrity": "sha1-VgiurfwAvmwpAd9fmGF4jeDVl8g="
},
"nan": {
"version": "2.8.0",
"resolved": "https://registry.npmjs.org/nan/-/nan-2.8.0.tgz",
"integrity": "sha1-7XFfP+neArV6XmJS2QqWZ14fCFo=",
"optional": true
},
"natural": {
"version": "0.5.6",
"resolved": "https://registry.npmjs.org/natural/-/natural-0.5.6.tgz",
"integrity": "sha1-8hUrYfNr78b+Dta8gR+0mebp3M0=",
"requires": {
"apparatus": "0.0.9",
"sylvester": "0.0.21",
"underscore": "1.8.3",
"webworker-threads": "0.7.13"
}
},
"negotiator": {
"version": "0.6.1",
"resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.1.tgz",
"integrity": "sha1-KzJxhOiZIQEXeyhWP7XnECrNDKk="
},
"node-xml-stream": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/node-xml-stream/-/node-xml-stream-1.0.2.tgz",
"integrity": "sha512-W4LzsKQ1x3TldQQmXZfM/JO+3SPp+zzvA2Nf82chE7vXnZy7LqHjIqQe9yDUw58g2fTOaJo+DlPnIDfmozgsuw=="
},
"oauth-sign": {
"version": "0.8.2",
"resolved": "https://registry.npmjs.org/oauth-sign/-/oauth-sign-0.8.2.tgz",
"integrity": "sha1-Rqarfwrq2N6unsBWV4C31O/rnUM="
},
"on-finished": {
"version": "2.3.0",
"resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz",
"integrity": "sha1-IPEzZIGwg811M3mSoWlxqi2QaUc=",
"requires": {
"ee-first": "1.1.1"
}
},
"parseurl": {
"version": "1.3.2",
"resolved": "https://registry.npmjs.org/parseurl/-/parseurl-1.3.2.tgz",
"integrity": "sha1-/CidTtiZMRlGDBViUyYs3I3mW/M="
},
"path-to-regexp": {
"version": "0.1.7",
"resolved": "https://registry.npmjs.org/path-to-regexp/-/path-to-regexp-0.1.7.tgz",
"integrity": "sha1-32BBeABfUi8V60SQ5yR6G/qmf4w="
},
"performance-now": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/performance-now/-/performance-now-2.1.0.tgz",
"integrity": "sha1-Ywn04OX6kT7BxpMHrjZLSzd8nns="
},
"process-nextick-args": {
"version": "1.0.7",
"resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-1.0.7.tgz",
"integrity": "sha1-FQ4gt1ZZCtP5EJPyWk8q2L/zC6M="
},
"progress": {
"version": "2.0.0",
"resolved": "https://registry.npmjs.org/progress/-/progress-2.0.0.tgz",
"integrity": "sha1-ihvjZr+Pwj2yvSPxDG/pILQ4nR8="
},
"proxy-addr": {
"version": "2.0.2",
"resolved": "https://registry.npmjs.org/proxy-addr/-/proxy-addr-2.0.2.tgz",
"integrity": "sha1-ZXFQT0e7mI7IGAJT+F3X4UlSvew=",
"requires": {
"forwarded": "0.1.2",
"ipaddr.js": "1.5.2"
}
},
"punycode": {
"version": "1.4.1",
"resolved": "https://registry.npmjs.org/punycode/-/punycode-1.4.1.tgz",
"integrity": "sha1-wNWmOycYgArY4esPpSachN1BhF4="
},
"qs": {
"version": "6.5.1",
"resolved": "https://registry.npmjs.org/qs/-/qs-6.5.1.tgz",
"integrity": "sha512-eRzhrN1WSINYCDCbrz796z37LOe3m5tmW7RQf6oBntukAG1nmovJvhnwHHRMAfeoItc1m2Hk02WER2aQ/iqs+A=="
},
"range-parser": {
"version": "1.2.0",
"resolved": "https://registry.npmjs.org/range-parser/-/range-parser-1.2.0.tgz",
"integrity": "sha1-9JvmtIeJTdxA3MlKMi9hEJLgDV4="
},
"raw-body": {
"version": "2.3.2",
"resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.3.2.tgz",
"integrity": "sha1-vNYMd9Prk83gBQKVw/N5OJvIj4k=",
"requires": {
"bytes": "3.0.0",
"http-errors": "1.6.2",
"iconv-lite": "0.4.19",
"unpipe": "1.0.0"
}
},
"readable-stream": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.3.tgz",
"integrity": "sha512-m+qzzcn7KUxEmd1gMbchF+Y2eIUbieUaxkWtptyHywrX0rE8QEYqPC07Vuy4Wm32/xE16NcdBctb8S0Xe/5IeQ==",
"requires": {
"core-util-is": "1.0.2",
"inherits": "2.0.3",
"isarray": "1.0.0",
"process-nextick-args": "1.0.7",
"safe-buffer": "5.1.1",
"string_decoder": "1.0.3",
"util-deprecate": "1.0.2"
}
},
"request": {
"version": "2.83.0",
"resolved": "https://registry.npmjs.org/request/-/request-2.83.0.tgz",
"integrity": "sha512-lR3gD69osqm6EYLk9wB/G1W/laGWjzH90t1vEa2xuxHD5KUrSzp9pUSfTm+YC5Nxt2T8nMPEvKlhbQayU7bgFw==",
"requires": {
"aws-sign2": "0.7.0",
"aws4": "1.6.0",
"caseless": "0.12.0",
"combined-stream": "1.0.5",
"extend": "3.0.1",
"forever-agent": "0.6.1",
"form-data": "2.3.1",
"har-validator": "5.0.3",
"hawk": "6.0.2",
"http-signature": "1.2.0",
"is-typedarray": "1.0.0",
"isstream": "0.1.2",
"json-stringify-safe": "5.0.1",
"mime-types": "2.1.17",
"oauth-sign": "0.8.2",
"performance-now": "2.1.0",
"qs": "6.5.1",
"safe-buffer": "5.1.1",
"stringstream": "0.0.5",
"tough-cookie": "2.3.3",
"tunnel-agent": "0.6.0",
"uuid": "3.2.1"
}
},
"safe-buffer": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.1.tgz",
"integrity": "sha512-kKvNJn6Mm93gAczWVJg7wH+wGYWNrDHdWvpUmHyEsgCtIwwo3bqPtV4tR5tuPaUhTOo/kvhVwd8XwwOllGYkbg=="
},
"sax": {
"version": "1.2.4",
"resolved": "https://registry.npmjs.org/sax/-/sax-1.2.4.tgz",
"integrity": "sha512-NqVDv9TpANUjFm0N8uM5GxL36UgKi9/atZw+x7YFnQ8ckwFGKrl4xX4yWtrey3UJm5nP1kUbnYgLopqWNSRhWw=="
},
"send": {
"version": "0.16.1",
"resolved": "https://registry.npmjs.org/send/-/send-0.16.1.tgz",
"integrity": "sha512-ElCLJdJIKPk6ux/Hocwhk7NFHpI3pVm/IZOYWqUmoxcgeyM+MpxHHKhb8QmlJDX1pU6WrgaHBkVNm73Sv7uc2A==",
"requires": {
"debug": "2.6.9",
"depd": "1.1.2",
"destroy": "1.0.4",
"encodeurl": "1.0.2",
"escape-html": "1.0.3",
"etag": "1.8.1",
"fresh": "0.5.2",
"http-errors": "1.6.2",
"mime": "1.4.1",
"ms": "2.0.0",
"on-finished": "2.3.0",
"range-parser": "1.2.0",
"statuses": "1.3.1"
}
},
"serve-static": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/serve-static/-/serve-static-1.13.1.tgz",
"integrity": "sha512-hSMUZrsPa/I09VYFJwa627JJkNs0NrfL1Uzuup+GqHfToR2KcsXFymXSV90hoyw3M+msjFuQly+YzIH/q0MGlQ==",
"requires": {
"encodeurl": "1.0.2",
"escape-html": "1.0.3",
"parseurl": "1.3.2",
"send": "0.16.1"
}
},
"setprototypeof": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.0.tgz",
"integrity": "sha512-BvE/TwpZX4FXExxOxZyRGQQv651MSwmWKZGqvmPcRIjDqWub67kTKuIMx43cZZrS/cBBzwBcNDWoFxt2XEFIpQ=="
},
"sntp": {
"version": "2.1.0",
"resolved": "https://registry.npmjs.org/sntp/-/sntp-2.1.0.tgz",
"integrity": "sha512-FL1b58BDrqS3A11lJ0zEdnJ3UOKqVxawAkF3k7F0CVN7VQ34aZrV+G8BZ1WC9ZL7NyrwsW0oviwsWDgRuVYtJg==",
"requires": {
"hoek": "4.2.0"
}
},
"sshpk": {
"version": "1.13.1",
"resolved": "https://registry.npmjs.org/sshpk/-/sshpk-1.13.1.tgz",
"integrity": "sha1-US322mKHFEMW3EwY/hzx2UBzm+M=",
"requires": {
"asn1": "0.2.3",
"assert-plus": "1.0.0",
"bcrypt-pbkdf": "1.0.1",
"dashdash": "1.14.1",
"ecc-jsbn": "0.1.1",
"getpass": "0.1.7",
"jsbn": "0.1.1",
"tweetnacl": "0.14.5"
}
},
"statuses": {
"version": "1.3.1",
"resolved": "https://registry.npmjs.org/statuses/-/statuses-1.3.1.tgz",
"integrity": "sha1-+vUbnrdKrvOzrPStX2Gr8ky3uT4="
},
"stemmer": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/stemmer/-/stemmer-1.0.2.tgz",
"integrity": "sha1-Ha/SoD01A1NvPhUy2nj77iKeACQ="
},
"string_decoder": {
"version": "1.0.3",
"resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.0.3.tgz",
"integrity": "sha512-4AH6Z5fzNNBcH+6XDMfA/BTt87skxqJlO0lAh3Dker5zThcAxG6mKz+iGu308UKoPPQ8Dcqx/4JhujzltRa+hQ==",
"requires": {
"safe-buffer": "5.1.1"
}
},
"stringstream": {
"version": "0.0.5",
"resolved": "https://registry.npmjs.org/stringstream/-/stringstream-0.0.5.tgz",
"integrity": "sha1-TkhM1N5aC7vuGORjB3EKioFiGHg="
},
"sylvester": {
"version": "0.0.21",
"resolved": "https://registry.npmjs.org/sylvester/-/sylvester-0.0.21.tgz",
"integrity": "sha1-KYexzivS84sNzio0OIiEv6RADqc="
},
"tough-cookie": {
"version": "2.3.3",
"resolved": "https://registry.npmjs.org/tough-cookie/-/tough-cookie-2.3.3.tgz",
"integrity": "sha1-C2GKVWW23qkL80JdBNVe3EdadWE=",
"requires": {
"punycode": "1.4.1"
}
},
"tunnel-agent": {
"version": "0.6.0",
"resolved": "https://registry.npmjs.org/tunnel-agent/-/tunnel-agent-0.6.0.tgz",
"integrity": "sha1-J6XeoGs2sEoKmWZ3SykIaPD8QP0=",
"requires": {
"safe-buffer": "5.1.1"
}
},
"tweetnacl": {
"version": "0.14.5",
"resolved": "https://registry.npmjs.org/tweetnacl/-/tweetnacl-0.14.5.tgz",
"integrity": "sha1-WuaBd/GS1EViadEIr6k/+HQ/T2Q=",
"optional": true
},
"type-is": {
"version": "1.6.15",
"resolved": "https://registry.npmjs.org/type-is/-/type-is-1.6.15.tgz",
"integrity": "sha1-yrEPtJCeRByChC6v4a1kbIGARBA=",
"requires": {
"media-typer": "0.3.0",
"mime-types": "2.1.17"
}
},
"typedarray": {
"version": "0.0.6",
"resolved": "https://registry.npmjs.org/typedarray/-/typedarray-0.0.6.tgz",
"integrity": "sha1-hnrHTjhkGHsdPUfZlqeOxciDB3c="
},
"underscore": {
"version": "1.8.3",
"resolved": "https://registry.npmjs.org/underscore/-/underscore-1.8.3.tgz",
"integrity": "sha1-Tz+1OxBuYJf8+ctBCfKl6b36UCI="
},
"unpipe": {
"version": "1.0.0",
"resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
"integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
},
"util-deprecate": {
"version": "1.0.2",
"resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
"integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8="
},
"utils-merge": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/utils-merge/-/utils-merge-1.0.1.tgz",
"integrity": "sha1-n5VxD1CiZ5R7LMwSR0HBAoQn5xM="
},
"uuid": {
"version": "3.2.1",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-3.2.1.tgz",
"integrity": "sha512-jZnMwlb9Iku/O3smGWvZhauCf6cvvpKi4BKRiliS3cxnI+Gz9j5MEpTz2UFuXiKPJocb7gnsLHwiS05ige5BEA=="
},
"vary": {
"version": "1.1.2",
"resolved": "https://registry.npmjs.org/vary/-/vary-1.1.2.tgz",
"integrity": "sha1-IpnwLG3tMNSllhsLn3RSShj2NPw="
},
"verror": {
"version": "1.10.0",
"resolved": "https://registry.npmjs.org/verror/-/verror-1.10.0.tgz",
"integrity": "sha1-OhBcoXBTr1XW4nDB+CiGguGNpAA=",
"requires": {
"assert-plus": "1.0.0",
"core-util-is": "1.0.2",
"extsprintf": "1.3.0"
}
},
"webworker-threads": {
"version": "0.7.13",
"resolved": "https://registry.npmjs.org/webworker-threads/-/webworker-threads-0.7.13.tgz",
"integrity": "sha1-yEsYtrokElu503NC5E3rgVFi+4M=",
"optional": true,
"requires": {
"bindings": "1.3.0",
"nan": "2.8.0"
}
},
"xml2object": {
"version": "0.1.2",
"resolved": "https://registry.npmjs.org/xml2object/-/xml2object-0.1.2.tgz",
"integrity": "sha1-hylkKI6BgaUP3UT3iRCX/lyYK0U=",
"requires": {
"sax": "1.2.4"
}
},
"zlib": {
"version": "1.0.5",
"resolved": "https://registry.npmjs.org/zlib/-/zlib-1.0.5.tgz",
"integrity": "sha1-bnyXL8NxxkWmr7A6sUdp3vEU/MA="
}
}
}

29
Class2/marc_template/package.json Normal file
View File

@ -0,0 +1,29 @@
{
"name": "LOC_MARC",
"version": "1.0.0",
"description": "LOC MARC Network Munge",
"main": "index.js",
"scripts": {
"start": "node index.js",
"download-data": "wget --input-file data/urls.txt --directory-prefix data"
},
"keywords": [
"libraries"
],
"author": "Jer Thorp",
"license": "ISC",
"dependencies": {
"app-root-path": "^2.0.1",
"concat-stream": "^1.6.0",
"en-stemmer": "^1.0.3",
"express": "^4.16.2",
"fs": "0.0.1-security",
"natural": "^0.5.4",
"node-xml-stream": "^1.0.2",
"progress": "^2.0.0",
"request": "^2.83.0",
"stemmer": "^1.0.2",
"xml2object": "^0.1.2",
"zlib": "^1.0.5"
}
}

11
Class2/marc_template/server.js Normal file
View File

@ -0,0 +1,11 @@
// init project
var express = require('express');
var app = express();
// http://expressjs.com/en/starter/static-files.html
app.use(express.static('public'));
// listen for requests :)
var listener = app.listen(process.env.PORT, function () {
console.log('Your app is listening on port ' + listener.address().port);
});