Skip to content

Commit c1df0d3

Browse files
authoredMar 16, 2019
fix: add author size limitation (#154)
Just for skipping false-hood values .cc @wooorm
1 parent 0ae78ab commit c1df0d3

File tree

12 files changed

+928
-627
lines changed

12 files changed

+928
-627
lines changed
 

‎package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -64,7 +64,7 @@
6464
"clean": "rm -rf node_modules && cd packages && eachdir rm -rf node_modules",
6565
"dev": "concurrently \"gulp\" \"npm run server:dev\"",
6666
"lint": "standard-markdown README.md && standard",
67-
"postinstall": "lerna bootstrap && lerna link --force-local",
67+
"postinstall": "lerna bootstrap --no-ci && lerna link --force-local",
6868
"prerelease": "npm run update:check",
6969
"pretest": "npm run lint",
7070
"preversion": "lerna exec npx finepack && git-authors-cli && finepack",

‎packages/metascraper-helpers/index.js

+6-2
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,9 @@ const {
1010
flow,
1111
chain,
1212
isEmpty,
13-
eq
13+
eq,
14+
lte,
15+
size
1416
} = require('lodash')
1517

1618
const langs = require('iso-639-3').map(({ iso6391 }) => iso6391)
@@ -61,6 +63,8 @@ const REGEX_LOCATION = /^[A-Z\s]+\s+[-—–]\s+/
6163

6264
const TRUNCATE_MAX_LENGTH = 300
6365

66+
const AUTHOR_MAX_LENGTH = 128
67+
6468
const removeLocation = value => replace(value, REGEX_LOCATION, '')
6569

6670
const isUrl = (url, { relative = false } = {}) =>
@@ -102,7 +106,7 @@ const $filter = ($, domNodes, fn = defaultFn) => {
102106
}
103107

104108
const isAuthor = (str, opts = { relative: false }) =>
105-
isString(str) && !isUrl(str, opts)
109+
!isUrl(str, opts) && isString(str) && lte(size(str), AUTHOR_MAX_LENGTH)
106110

107111
const getAuthor = (str, opts = { removeBy: true }) => titleize(str, opts)
108112

‎packages/metascraper-helpers/package.json

+3-3
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@
1717
],
1818
"dependencies": {
1919
"audio-extensions": "0.0.0",
20-
"chrono-node": "~1.3.5",
20+
"chrono-node": "~1.3.11",
2121
"condense-whitespace": "~1.0.0",
2222
"file-extension": "~4.0.5",
2323
"image-extensions": "~1.1.0",
@@ -26,8 +26,8 @@
2626
"iso-639-3": "~1.1.0",
2727
"isostring": "0.0.1",
2828
"lodash": "~4.17.11",
29-
"mime-types": "~2.1.21",
30-
"normalize-url": "~4.1.0",
29+
"mime-types": "~2.1.22",
30+
"normalize-url": "~4.2.0",
3131
"smartquotes": "~2.3.1",
3232
"title": "~3.4.1",
3333
"truncate": "~2.0.1",

‎packages/metascraper-media-provider/package.json

+1-1
Original file line numberDiff line numberDiff line change
@@ -18,7 +18,7 @@
1818
],
1919
"dependencies": {
2020
"@metascraper/helpers": "^4.10.1",
21-
"@microlink/youtube-dl": "~1.13.0",
21+
"@microlink/youtube-dl": "~1.13.1",
2222
"got": "~9.6.0",
2323
"lodash": "~4.17.11",
2424
"memoize-one": "~5.0.0",

‎packages/metascraper-media-provider/src/get-media/twitter-info.js

+2-2
Original file line numberDiff line numberDiff line change
@@ -28,7 +28,7 @@ const API_GUEST_ACTIVATE_EXPIRE = 10 * 60 * 1000 // 10 min
2828

2929
const { PROXY_HOST, PROXY_PORT, PROXY_USER, PROXY_PASS } = process.env
3030

31-
let agent = PROXY_HOST
31+
const agent = PROXY_HOST
3232
? tunnel.httpsOverHttp({
3333
proxy: {
3434
host: PROXY_HOST,
@@ -43,7 +43,7 @@ const getGuestToken = async (url = '', opts = {}) => {
4343
const { body } = await got.post(
4444
'https://api.twitter.com/1.1/guest/activate.json',
4545
{
46-
headers: { Authorization: TWITTER_BEARER_TOKEN, Referer: url },
46+
headers: { Authorization: TWITTER_BEARER_TOKEN },
4747
json: true,
4848
retry: 0,
4949
agent,

‎packages/metascraper-media-provider/test/index.js

+1-1
Original file line numberDiff line numberDiff line change
@@ -84,7 +84,7 @@ describe('metascraper-media-provider', () => {
8484
;[
8585
'https://twitter.com/verge/status/957383241714970624',
8686
'https://twitter.com/telediario_tve/status/1036860275859775488',
87-
'https://twitter.com/Mei_Gui8/status/1037374230785142785'
87+
'https://twitter.com/futurism/status/882987478541533189'
8888
].forEach(url => {
8989
it(url, async () => {
9090
const metadata = await metascraper({ url })

‎packages/metascraper-youtube/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
],
1919
"dependencies": {
2020
"@metascraper/helpers": "^4.10.1",
21-
"get-video-id": "~3.1.0",
21+
"get-video-id": "~3.1.1",
2222
"is-reachable": "~3.0.0",
23-
"p-locate": "~3.0.0"
23+
"p-locate": "~4.0.0"
2424
},
2525
"devDependencies": {
2626
"mocha": "latest",

‎packages/metascraper/__snapshots__/index.js.snap-shot

+13-13
Original file line numberDiff line numberDiff line change
@@ -206,19 +206,6 @@ exports['entrepreneur 1'] = {
206206
"url": "https://www.entrepreneur.com/article/275188"
207207
}
208208

209-
exports['et-tech 1'] = {
210-
"author": "TV Mahalingam",
211-
"date": "2016-05-25T00:00:00.000Z",
212-
"description": "Dell speaks about the complexities of engineering the biggest acquisition in tech history, Dell’s future bets and why he continues to remain bullish..",
213-
"image": "http://economictimes.indiatimes.com/photo/michael-dell-on-complexities-involved-in-pulling-off-the-biggest-tech-acquisition-dell-s-future-bets-more/52424992.cms",
214-
"video": null,
215-
"lang": null,
216-
"logo": "http://b2bstatic.iimg.in/Themes/Release/images/responsive/tech-logo-square.jpg",
217-
"publisher": "ETtech.com",
218-
"title": "Michael Dell on complexities involved in pulling off the biggest tech acquisition, Dell’s future bets & more | ETtech",
219-
"url": "http://tech.economictimes.indiatimes.com/news/corporate/michael-dell-biggest-tech-acquisition-future-bets/52424992"
220-
}
221-
222209
exports['eweek 1'] = {
223210
"author": "Darryl K. Taft",
224211
"date": "2019-01-09T09:01:37.000Z",
@@ -994,3 +981,16 @@ exports['engadget 1'] = {
994981
"url": "https://www.engadget.com/2019/01/07/all-github-users-keep-code-private/"
995982
}
996983

984+
exports['learnnode 1'] = {
985+
"author": null,
986+
"date": null,
987+
"description": "A premium training course to learn to build apps with Node.js, Express, MongoDB, and friends.",
988+
"image": "https://learnnode.com/images/NODE/poster.jpg",
989+
"video": "https://player.vimeo.com/external/216213305.sd.mp4?s=815e208b400abe120e9b860dad68762bcf4b828a&profile_id=164",
990+
"lang": null,
991+
"logo": "https://learnnode.com/images/NODE/favicon.png",
992+
"publisher": "Learn Node",
993+
"title": "Learn Node",
994+
"url": "https://learnnode.com"
995+
}
996+

‎packages/metascraper/package.json

+2-2
Original file line numberDiff line numberDiff line change
@@ -55,8 +55,8 @@
5555
"cheerio": "~1.0.0-rc.2",
5656
"cheerio-advanced-selectors": "~2.0.1",
5757
"lodash": "~4.17.11",
58-
"p-reduce": "~1.0.0",
59-
"whoops": "~4.0.1"
58+
"p-reduce": "~2.0.0",
59+
"whoops": "~4.0.2"
6060
},
6161
"devDependencies": {
6262
"clear-module": "latest",

0 commit comments

Comments
 (0)