Fix: Corrige mensagem de erro de IP bloqueado no VideoMind quando o vídeo simplesmente não possui legendas

2026-06-05 20:56:39 +00:00
parent 7a2ba56699
commit 281005a8bc
1086 changed files with 46356 additions and 7 deletions
@@ -0,0 +1,7 @@
+{
+  "presets": [
+    ["env", { "targets": { "node": "6" } }], 
+    "flow",
+    "stage-0"
+  ]
+}
@@ -0,0 +1,3 @@
+module.exports = {
+  extends: 'algolia/flowtype'
+}
@@ -0,0 +1,11 @@
+[ignore]
+
+[include]
+
+[libs]
+
+[lints]
+
+[options]
+
+[strict]
@@ -0,0 +1,15 @@
+# v2.0.2
+
+Add support for browser extensions via conditionally using fetch over axios
+
+# v2.0.1
+
+Fixed Invalid attempt to destructure non-iterable instance error
+
+# v2.0.0
+
+Use youtube.com/watch instead of /getinfo
+
+# v1.0.1
+
+* strip HTML tags from captions
@@ -0,0 +1,42 @@
+# Youtube Captions scraper
+
+> Fetch youtube user submitted or fallback to auto-generated captions
+
+## Installation
+
+* `> npm install -S youtube-captions-scraper` OR
+* `> yarn add youtube-captions-scraper`
+
+## Usage
+
+```js
+// ES6 / TypeScript
+import { getSubtitles } from 'youtube-captions-scraper';
+
+getSubtitles({
+  videoID: 'XXXXX', // youtube video id
+  lang: 'fr' // default: `en`
+}).then(captions => {
+  console.log(captions);
+});
+
+// ES5
+var getSubtitles = require('youtube-captions-scraper').getSubtitles;
+
+getSubtitles({
+  videoID: 'XXXXX', // youtube video id
+  lang: 'fr' // default: `en`
+}).then(function(captions) {
+  console.log(captions);
+});
+```
+
+Captions will be an array of object of this format:
+
+```js
+{
+  "start": Number,
+  "dur": Number,
+  "text": String
+}
+```
@@ -0,0 +1,123 @@
+'use strict';
+
+Object.defineProperty(exports, "__esModule", {
+  value: true
+});
+exports.getSubtitles = undefined;
+
+var _slicedToArray = function () { function sliceIterator(arr, i) { var _arr = []; var _n = true; var _d = false; var _e = undefined; try { for (var _i = arr[Symbol.iterator](), _s; !(_n = (_s = _i.next()).done); _n = true) { _arr.push(_s.value); if (i && _arr.length === i) break; } } catch (err) { _d = true; _e = err; } finally { try { if (!_n && _i["return"]) _i["return"](); } finally { if (_d) throw _e; } } return _arr; } return function (arr, i) { if (Array.isArray(arr)) { return arr; } else if (Symbol.iterator in Object(arr)) { return sliceIterator(arr, i); } else { throw new TypeError("Invalid attempt to destructure non-iterable instance"); } }; }();
+
+let getSubtitles = exports.getSubtitles = (() => {
+  var _ref4 = _asyncToGenerator(function* ({
+    videoID,
+    lang = 'en'
+  }) {
+    const data = yield fetchData(`https://youtube.com/watch?v=${videoID}`);
+
+    // * ensure we have access to captions data
+    if (!data.includes('captionTracks')) throw new Error(`Could not find captions for video: ${videoID}`);
+
+    const regex = /"captionTracks":(\[.*?\])/;
+
+    var _regex$exec = regex.exec(data),
+        _regex$exec2 = _slicedToArray(_regex$exec, 1);
+
+    const match = _regex$exec2[0];
+
+    var _JSON$parse = JSON.parse(`{${match}}`);
+
+    const captionTracks = _JSON$parse.captionTracks;
+
+    const subtitle = (0, _lodash.find)(captionTracks, {
+      vssId: `.${lang}`
+    }) || (0, _lodash.find)(captionTracks, {
+      vssId: `a.${lang}`
+    }) || (0, _lodash.find)(captionTracks, function ({ vssId }) {
+      return vssId && vssId.match(`.${lang}`);
+    });
+
+    // * ensure we have found the correct subtitle lang
+    if (!subtitle || subtitle && !subtitle.baseUrl) throw new Error(`Could not find ${lang} captions for ${videoID}`);
+
+    const transcript = yield fetchData(subtitle.baseUrl);
+    const lines = transcript.replace('<?xml version="1.0" encoding="utf-8" ?><transcript>', '').replace('</transcript>', '').split('</text>').filter(function (line) {
+      return line && line.trim();
+    }).map(function (line) {
+      const startRegex = /start="([\d.]+)"/;
+      const durRegex = /dur="([\d.]+)"/;
+
+      var _startRegex$exec = startRegex.exec(line),
+          _startRegex$exec2 = _slicedToArray(_startRegex$exec, 2);
+
+      const start = _startRegex$exec2[1];
+
+      var _durRegex$exec = durRegex.exec(line),
+          _durRegex$exec2 = _slicedToArray(_durRegex$exec, 2);
+
+      const dur = _durRegex$exec2[1];
+
+
+      const htmlText = line.replace(/<text.+>/, '').replace(/&amp;/gi, '&').replace(/<\/?[^>]+(>|$)/g, '');
+
+      const decodedText = _he2.default.decode(htmlText);
+      const text = (0, _striptags2.default)(decodedText);
+
+      return {
+        start,
+        dur,
+        text
+      };
+    });
+
+    return lines;
+  });
+
+  return function getSubtitles(_x3) {
+    return _ref4.apply(this, arguments);
+  };
+})();
+
+var _he = require('he');
+
+var _he2 = _interopRequireDefault(_he);
+
+var _axios = require('axios');
+
+var _axios2 = _interopRequireDefault(_axios);
+
+var _lodash = require('lodash');
+
+var _striptags = require('striptags');
+
+var _striptags2 = _interopRequireDefault(_striptags);
+
+function _interopRequireDefault(obj) { return obj && obj.__esModule ? obj : { default: obj }; }
+
+function _asyncToGenerator(fn) { return function () { var gen = fn.apply(this, arguments); return new Promise(function (resolve, reject) { function step(key, arg) { try { var info = gen[key](arg); var value = info.value; } catch (error) { reject(error); return; } if (info.done) { resolve(value); } else { return Promise.resolve(value).then(function (value) { step("next", value); }, function (err) { step("throw", err); }); } } return step("next"); }); }; }
+
+const fetchData = typeof fetch === 'function' ? (() => {
+  var _ref = _asyncToGenerator(function* (url) {
+    const response = yield fetch(url);
+    return yield response.text();
+  });
+
+  function fetchData(_x) {
+    return _ref.apply(this, arguments);
+  }
+
+  return fetchData;
+})() : (() => {
+  var _ref2 = _asyncToGenerator(function* (url) {
+    var _ref3 = yield _axios2.default.get(url);
+
+    const data = _ref3.data;
+
+    return data;
+  });
+
+  function fetchData(_x2) {
+    return _ref2.apply(this, arguments);
+  }
+
+  return fetchData;
+})();
@@ -0,0 +1,58 @@
+{
+  "name": "youtube-captions-scraper",
+  "version": "2.0.3",
+  "description": "Scrap youtube auto-generated captions",
+  "main": "dist/index.js",
+  "author": {
+    "name": "Algolia, Inc.",
+    "url": "https://www.algolia.com"
+  },
+  "repository": {
+    "type": "git",
+    "url": "https://github.com/algolia/youtube-captions-scraper.git"
+  },
+  "homepage": "https://github.com/algolia/youtube-captions-scraper",
+  "bugs": {
+    "url": "https://github.com/algolia/youtube-captions-scraper/issues"
+  },
+  "license": "MIT",
+  "scripts": {
+    "build": "rm -rf dist && babel src -d dist",
+    "prepublishOnly": "npm run build",
+    "lint": "eslint src",
+    "test": "ava",
+    "flow": "flow"
+  },
+  "devDependencies": {
+    "@babel/cli": "^7.23.0",
+    "@babel/core": "^7.23.3",
+    "ava": "^0.25.0",
+    "babel-cli": "^6.26.0",
+    "babel-eslint": "^8.0.2",
+    "babel-preset-env": "^1.6.1",
+    "babel-preset-flow": "^6.23.0",
+    "babel-preset-stage-0": "^6.24.1",
+    "babel-watch": "^2.0.7",
+    "eslint": "^4.11.0",
+    "eslint-config-algolia": "^12.0.0",
+    "eslint-config-prettier": "^2.8.0",
+    "eslint-plugin-flowtype": "^2.39.1",
+    "eslint-plugin-import": "^2.8.0",
+    "eslint-plugin-prettier": "^2.3.1",
+    "flow-bin": "^0.59.0",
+    "flow-typed": "^2.2.3",
+    "prettier": "^1.8.2"
+  },
+  "dependencies": {
+    "axios": "^1.6.7",
+    "he": "^1.1.1",
+    "lodash": "^4.17.4",
+    "striptags": "^3.1.0"
+  },
+  "ava": {
+    "babel": "inherit",
+    "require": [
+      "babel-register"
+    ]
+  }
+}
@@ -0,0 +1,80 @@
+/* @flow */
+
+import he from 'he';
+import axios from 'axios';
+import { find } from 'lodash';
+import striptags from 'striptags';
+
+const fetchData =
+  typeof fetch === 'function'
+    ? async function fetchData(url) {
+        const response = await fetch(url);
+        return await response.text();
+      }
+    : async function fetchData(url) {
+        const { data } = await axios.get(url);
+        return data;
+      };
+
+export async function getSubtitles({
+  videoID,
+  lang = 'en',
+}: {
+  videoID: string,
+  lang: 'en' | 'de' | 'fr' | void,
+}) {
+  const data = await fetchData(
+    `https://youtube.com/watch?v=${videoID}`
+  );
+
+  // * ensure we have access to captions data
+  if (!data.includes('captionTracks'))
+    throw new Error(`Could not find captions for video: ${videoID}`);
+
+  const regex =  /"captionTracks":(\[.*?\])/;
+  const [match] = regex.exec(data);
+
+  const { captionTracks } = JSON.parse(`{${match}}`);
+  const subtitle =
+    find(captionTracks, {
+      vssId: `.${lang}`,
+    }) ||
+    find(captionTracks, {
+      vssId: `a.${lang}`,
+    }) ||
+    find(captionTracks, ({ vssId }) => vssId && vssId.match(`.${lang}`));
+
+  // * ensure we have found the correct subtitle lang
+  if (!subtitle || (subtitle && !subtitle.baseUrl))
+    throw new Error(`Could not find ${lang} captions for ${videoID}`);
+
+  const transcript = await fetchData(subtitle.baseUrl);
+  const lines = transcript
+    .replace('<?xml version="1.0" encoding="utf-8" ?><transcript>', '')
+    .replace('</transcript>', '')
+    .split('</text>')
+    .filter(line => line && line.trim())
+    .map(line => {
+      const startRegex = /start="([\d.]+)"/;
+      const durRegex = /dur="([\d.]+)"/;
+
+      const [, start] = startRegex.exec(line);
+      const [, dur] = durRegex.exec(line);
+
+      const htmlText = line
+        .replace(/<text.+>/, '')
+        .replace(/&amp;/gi, '&')
+        .replace(/<\/?[^>]+(>|$)/g, '');
+
+      const decodedText = he.decode(htmlText);
+      const text = striptags(decodedText);
+
+      return {
+        start,
+        dur,
+        text,
+      };
+    });
+
+  return lines;
+}
@@ -0,0 +1,16 @@
+import test from 'ava';
+import { getSubtitles } from '../src';
+
+test('Extract estonia war subtitles', async t => {
+  const subtitles = await getSubtitles({ videoID: 'HBA0xDHZjko' });
+  t.deepEqual(subtitles[0], {
+    dur: '4.72',
+    start: '6.98',
+    text: 'November 1918',
+  });
+});
+
+test('Extract passive income video', async t => {
+  const subtitles = await getSubtitles({ videoID: 'JueUvj6X3DA' });
+  t.deepEqual('creating passive income takes work but', subtitles[0].text);
+});