diff --git a/.env.example b/.env.example index 783ce16..3dc9495 100644 --- a/.env.example +++ b/.env.example @@ -1,7 +1,5 @@ EMAIL=test.com PASSWORD=password -ALGOLIA_APP_ID= -ALGOLIA_API_KEY= LOCAL_PATH=Downloads LESSONS_FOLDER=lessons SERIES_FOLDER=series diff --git a/.gitignore b/.gitignore index 95413e1..8293591 100644 --- a/.gitignore +++ b/.gitignore @@ -15,3 +15,4 @@ App/.DS_Store .DS_Store .phpintel/ +Downloads/cache.php diff --git a/App/Algolia/Controller.php b/App/Algolia/Controller.php deleted file mode 100644 index c797fc5..0000000 --- a/App/Algolia/Controller.php +++ /dev/null @@ -1,99 +0,0 @@ -client = $client; - } - - /** - * Grabs all lessons & series from the algolia api. - * - * @return array - * @throws \AlgoliaSearch\AlgoliaException - */ - public function getAllLessons() - { - $index = $this->client->initIndex(ALGOLIA_INDEX_NAME); - - $page = 0; - $params = [ - 'facetFilters' => [ - [ - 'type:lesson', - 'type:series', - ], - ], - 'attributesToRetrieve' => [ - 'path', - 'type', - 'slug', - 'episode_count', - ], - 'page' => $page, - ]; - - $array = [ - 'lessons' => [], - 'series' => [], - ]; - - do { - try { - $res = $index->search('', $params); - } catch (\Exception $e) { - throw new AlgoliaException($e->getMessage(), $e->getCode(), $e); - } - $params['page'] = $res['page'] + 1; - - foreach ($res['hits'] as $lessonInfo) { - switch ($lessonInfo['type']) { - case 'lesson': - $path = $lessonInfo['path']; - if (preg_match('/'.LARACASTS_LESSONS_PATH.'\/(.+)/', $path, $matches)) { // lesson - $array['lessons'][] = $matches[1]; - } - break; - case 'series': - $serieSlug = $lessonInfo['slug']; - foreach (range(1, $lessonInfo['episode_count']) as $episode) { - $array['series'][$serieSlug][] = $episode; - } - break; - default: - break; - } - } - - } while ($res['page'] <= $res['nbPages']); - - Downloader::$currentLessonNumber = count($array['lessons']); - - return $array; - } - -} diff --git a/App/Downloader.php b/App/Downloader.php index a76cba1..b86b21e 100644 --- a/App/Downloader.php +++ b/App/Downloader.php @@ -2,69 +2,52 @@ /** * Main cycle of the app */ + namespace App; use App\Exceptions\LoginException; -use App\Exceptions\SubscriptionNotActiveException; use App\Http\Resolver; +use App\Laracasts\Controller as LaracastsController; use App\System\Controller as SystemController; use App\Utils\Utils; use Cocur\Slugify\Slugify; use GuzzleHttp\Client as HttpClient; -use AlgoliaSearch\Client as AlgoliaClient; -use App\Algolia\Controller as AlgoliaController; -use App\Laracasts\Controller as LaracastsController; use League\Flysystem\Filesystem; use Ubench; /** * Class Downloader + * * @package App */ class Downloader { /** * Http resolver object + * * @var Resolver */ private $client; /** * System object + * * @var SystemController */ private $system; /** * Ubench lib + * * @var Ubench */ private $bench; - /** - * Algolia object - * @var AlgoliaController - */ - private $algolia; - - /** - * Number of local lessons - * @var int - */ - public static $totalLocalLessons; - - /** - * Current lesson number - * @var int - */ - public static $currentLessonNumber; - private $wantSeries = []; - private $wantLessons = []; - //this filters all online episodes to what the user has requested - // this filter is applied before checking local (downloaded) videos, - // so local videos exclusion will work as before + // this filters all online episodes to what the user has requested + // this filter is applied before checking local (downloaded) videos, + // so local videos exclusion will work as before private $filterSeriesEpisodes = []; /** @@ -78,16 +61,14 @@ class Downloader * @param HttpClient $httpClient * @param Filesystem $system * @param Ubench $bench - * @param AlgoliaClient $algoliaClient * @param bool $retryDownload */ - public function __construct(HttpClient $httpClient, Filesystem $system, Ubench $bench, AlgoliaClient $algoliaClient, $retryDownload = false) + public function __construct(HttpClient $httpClient, Filesystem $system, Ubench $bench, $retryDownload = false) { $this->client = new Resolver($httpClient, $bench, $retryDownload); $this->system = new SystemController($system); $this->bench = $bench; - $this->algolia = new AlgoliaController($algoliaClient); - $this->laracasts = new LaracastsController($httpClient); + $this->laracasts = new LaracastsController($this->client); } /** @@ -97,111 +78,92 @@ public function __construct(HttpClient $httpClient, Filesystem $system, Ubench $ */ public function start($options) { - try { - $counter = [ - 'series' => 1, - 'lessons' => 1, - 'failed_episode' => 0, - 'failed_lesson' => 0 - ]; - - Utils::box('Authenticating'); + $counter = [ + 'series' => 1, + 'failed_episode' => 0, + ]; - $this->doAuth($options); + $this->authenticate($options['email'], $options['password']); - Utils::box('Starting Collecting the data'); + Utils::box('Starting Collecting the data'); - $this->bench->start(); + $this->bench->start(); - $localLessons = $this->system->getAllLessons(); - $algoliaLessons = $this->algolia->getAllLessons(); - $this->laracasts->addAlgoliaResults($algoliaLessons); - $allLessonsOnline = $this->laracasts->getAllSeries(); + $localSeries = $this->system->getSeries(); - $this->bench->end(); + $cachedData = $this->system->getCache(); - if($this->_haveOptions()) { //filter all online lessons to the selected ones - $allLessonsOnline = $this->onlyDownloadProvidedLessonsAndSeries($allLessonsOnline); - } + $onlineSeries = $this->laracasts->getSeries($cachedData); - Utils::box('Downloading'); - //Magic to get what to download - $diff = Utils::resolveFaultyLessons($allLessonsOnline, $localLessons); + $this->system->setCache($onlineSeries); - $new_lessons = Utils::countLessons($diff); - $new_episodes = Utils::countEpisodes($diff); + $this->bench->end(); - Utils::write(sprintf("%d new lessons and %d episodes. %s elapsed with %s of memory usage.", - $new_lessons, - $new_episodes, - $this->bench->getTime(), - $this->bench->getMemoryUsage()) - ); + if ($this->_haveOptions()) { //filter all online lessons to the selected ones + $onlineSeries = $this->onlyDownloadProvidedSeries($onlineSeries); //TODO: Check + } + Utils::box('Downloading'); + //Magic to get what to download + $diff = Utils::compareLocalAndOnlineSeries($onlineSeries, $localSeries); - //Download Lessons - if ($new_lessons > 0) { - $this->downloadLessons($diff, $counter, $new_lessons); - } + $new_episodes = Utils::countEpisodes($diff); - //Donwload Episodes - if ($new_episodes > 0) { - $this->downloadEpisodes($diff, $counter, $new_episodes); - } + Utils::write(sprintf("%d new episodes. %s elapsed with %s of memory usage.", + $new_episodes, + $this->bench->getTime(), + $this->bench->getMemoryUsage()) + ); - Utils::writeln(sprintf("Finished! Downloaded %d new lessons and %d new episodes. Failed: %d", - $new_lessons - $counter['failed_lesson'], - $new_episodes - $counter['failed_episode'], - $counter['failed_lesson'] + $counter['failed_episode'] - )); - } catch (LoginException $e) { - Utils::write("Your login details are wrong!"); - } catch (SubscriptionNotActiveException $e) { - Utils::write('Your subscription is not active!'); + //Download Episodes + if ($new_episodes > 0) { + $this->downloadEpisodes($diff, $counter, $new_episodes); } + + Utils::writeln(sprintf("Finished! Downloaded %d new episodes. Failed: %d", + $new_episodes - $counter['failed_episode'], + $counter['failed_episode'] + )); } /** * Tries to login. * - * @param $options - * - * @throws \Exception + * @param string $email + * @param string $password + * @return bool + * @throws LoginException */ - public function doAuth($options) + public function authenticate($email, $password) { - if (!$this->client->doAuth($options['email'], $options['password'])) { - throw new LoginException("Can't do the login.."); + Utils::box('Authenticating'); + + if (empty($email) and empty($password)) { + Utils::write("No EMAIL and PASSWORD is set in .env file"); + Utils::write("Browsing as guest and can only download free lessons."); + + return false; } - Utils::write("Successfull!"); - } - /** - * Download Lessons - * @param $diff - * @param $counter - * @param $new_lessons - */ - public function downloadLessons(&$diff, &$counter, $new_lessons) - { - $this->system->createFolderIfNotExists(LESSONS_FOLDER); - Utils::box('Downloading Lessons'); - foreach ($diff['lessons'] as $lesson) { + $user = $this->client->login($email, $password); - if($this->client->downloadLesson($lesson) === false) { - $counter['failed_lesson']++; - } + if (! is_null($user['error'])) + throw new LoginException($user['error']); + + if ($user['signedIn']) + Utils::write("Logged in as " . $user['data']['email']); - Utils::write(sprintf("Current: %d of %d total. Left: %d", - $counter['lessons']++, - $new_lessons, - $new_lessons - $counter['lessons'] + 1 - )); + if (! $user['data']['subscribed']) { + Utils::write("You don't have active subscription!"); + Utils::write("You can only download free lessons."); } + + return $user['signedIn']; } /** * Download Episodes + * * @param $diff * @param $counter * @param $new_episodes @@ -209,13 +171,16 @@ public function downloadLessons(&$diff, &$counter, $new_lessons) public function downloadEpisodes(&$diff, &$counter, $new_episodes) { $this->system->createFolderIfNotExists(SERIES_FOLDER); + Utils::box('Downloading Series'); - foreach ($diff['series'] as $serie => $episodes) { - $this->system->createSerieFolderIfNotExists($serie); - foreach ($episodes as $episode) { - if($this->client->downloadSerieEpisode($serie, $episode) === false) { - $counter['failed_episode'] = $counter['failed_episode'] +1; + foreach ($diff as $serie) { + $this->system->createSerieFolderIfNotExists($serie['slug']); + + foreach ($serie['episodes'] as $episode) { + + if ($this->client->downloadEpisode($serie['slug'], $episode) === false) { + $counter['failed_episode'] = $counter['failed_episode'] + 1; } Utils::write(sprintf("Current: %d of %d total. Left: %d", @@ -227,80 +192,61 @@ public function downloadEpisodes(&$diff, &$counter, $new_episodes) } } + //TODO: Bug: occurs when using double -e option protected function _haveOptions() { $found = false; $short_options = "s:"; - $short_options .= "l:"; $short_options .= 'e:'; - $long_options = array( + $long_options = array( "series-name:", - "lesson-name:", "series-episodes:" ); $options = getopt($short_options, $long_options); - + Utils::box(sprintf("Checking for options %s", json_encode($options))); - - if(count($options) == 0) { + + if (count($options) == 0) { Utils::write('No options provided'); return false; } - + $slugify = new Slugify(); $slugify->addRule("'", ''); - - if(isset($options['s']) || isset($options['series-name'])) { + + if (isset($options['s']) || isset($options['series-name'])) { $series = isset($options['s']) ? $options['s'] : $options['series-name']; - if(!is_array($series)) - $series = [$series]; - + if (! is_array($series)) + $series = [$series]; + Utils::write(sprintf("Series names provided: %s", json_encode($series))); - - - $this->wantSeries = array_map(function ($serie) use ($slugify) { + + $this->wantSeries = array_map(function($serie) use ($slugify) { return $slugify->slugify($serie); }, $series); - - Utils::write(sprintf("Series names provided: %s", json_encode($this->wantSeries))); + Utils::write(sprintf("Series names provided: %s", json_encode($this->wantSeries))); - if(isset($options['e']) || isset($options['series-episodes'])) { + if (isset($options['e']) || isset($options['series-episodes'])) { $episodes = isset($options['e']) ? $options['e'] : $options['series-episodes']; - + Utils::write(sprintf("Episode numbers provided: %s", json_encode($episodes))); - if(strpos($episodes, ',') === false){ - if(!is_array($episodes)) + + if (strpos($episodes, ',') === false) { + if (! is_array($episodes)) $episodes = [$episodes]; - }else{ + } else { $episodes = explode(',', $episodes); } - + sort($episodes, SORT_NUMERIC); + $this->filterSeriesEpisodes = $episodes; - + Utils::write(sprintf("Episode numbers provided: %s", json_encode($this->filterSeriesEpisodes))); - } - - $found = true; - } - - if(isset($options['l']) || isset($options['lesson-name'])) { - $lessons = isset($options['l']) ? $options['l'] : $options['lesson-name']; - - if(!is_array($lessons)) - $lessons = [$lessons]; - - Utils::write(sprintf("Lesson names provided: %s", json_encode($lessons))); - - $this->wantLessons = array_map(function($lesson) use ($slugify) { - return $slugify->slugify($lesson); },$lessons - ); - - Utils::write(sprintf("Lesson names provided: %s", json_encode($this->wantLessons))); $found = true; } @@ -309,40 +255,32 @@ protected function _haveOptions() } /** - * Download selected Series and lessons - * @param $allLessonsOnline + * Download selected Series + * + * @param $onlineSeries * @return array */ - public function onlyDownloadProvidedLessonsAndSeries($allLessonsOnline) + public function onlyDownloadProvidedSeries($onlineSeries) { - Utils::box('Checking if series and lessons exists'); + Utils::box('Checking if series exists'); - $selectedLessonsOnline = [ - 'lessons' => [], - 'series' => [] - ]; + $selectedSeries = []; - foreach($this->wantSeries as $series) { - if(isset($allLessonsOnline['series'][$series])) { - Utils::write('Series "'.$series.'" found!'); - $selectedLessonsOnline['series'][$series] = $allLessonsOnline['series'][$series]; - if(is_array($this->filterSeriesEpisodes) && count($this->filterSeriesEpisodes) > 0){ - $selectedLessonsOnline['series'][$series] = $this->filterSeriesEpisodes; - } - } else { - Utils::write("Series '".$series."' not found!"); - } - } + foreach ($this->wantSeries as $serieSlug) { + if (isset($onlineSeries[$serieSlug])) { + Utils::write('Series "' . $serieSlug . '" found!'); - foreach($this->wantLessons as $lesson) { - if(in_array($lesson, $allLessonsOnline['lessons'])) { - Utils::write('Lesson "'.$lesson.'" found'); - $selectedLessonsOnline['lessons'][] = $lesson; + $selectedSeries[$serieSlug] = $onlineSeries[$serieSlug]; + + //TODO: Need to figure it how to handle it + /*if (is_array($this->filterSeriesEpisodes) && count($this->filterSeriesEpisodes) > 0) { + $selectedSeries[$serieSlug] = $this->filterSeriesEpisodes; + }*/ } else { - Utils::write("Lesson '".$lesson."' not found!"); + Utils::write("Series '" . $serieSlug . "' not found!"); } } - - return $selectedLessonsOnline; + + return $selectedSeries; } } diff --git a/App/Exceptions/AlgoliaException.php b/App/Exceptions/AlgoliaException.php deleted file mode 100644 index 40becc6..0000000 --- a/App/Exceptions/AlgoliaException.php +++ /dev/null @@ -1,27 +0,0 @@ -filter("input[name=_token]")->attr('value'); + return array_map(function($topic) { + return [ + 'slug' => str_replace(LARACASTS_BASE_URL . '/topics/', '', $topic['path']), + 'path' => $topic['path'], + 'episode_count' => $topic['episode_count'], + 'series_count' => $topic['series_count'] + ]; + }, $data['props']['topics']); } /** - * Gets the download link. + * Return full list of series for given topic HTML page. * - * @param $html - * @return string - * @throws NoDownloadLinkException + * @param string $html + * @return array */ - public static function getDownloadLink($html) + public static function getSeriesData($html) { - preg_match("(\"\/downloads\/.*?\")", $html, $matches); - - if(isset($matches[0]) === false) { - throw new NoDownloadLinkException(); - } - - return LARACASTS_BASE_URL . substr($matches[0],1,-1); + $data = self::getData($html); + + $series = $data['props']['topic']['series']; + + return array_combine( + array_column($series, 'slug'), + array_map(function($serie) { + return [ + 'slug' => $serie['slug'], + 'path' => LARACASTS_BASE_URL . $serie['path'], + 'episode_count' => $serie['episodeCount'], + 'is_complete' => $serie['complete'] + ]; + }, $series) + ); } /** - * Determine if this episode is scheduled for the future. + * Return full list of episodes for given series HTML page. * - * @param $html - * @return boolean + * @param string $html + * @return array */ - public static function scheduledEpisode($html) + public static function getEpisodesData($html) { - preg_match("(return to watch it (.*)\.)", $html, $matches); + $data = self::getData($html); + + $episodes = []; + + $chapters = $data['props']['series']['chapters']; - if (isset($matches[1])) { - return strip_tags($matches[1]); + foreach ($chapters as $chapter) { + foreach ($chapter['episodes'] as $episode) { + array_push($episodes, $episode); + } } - return false; + return array_filter( + array_combine( + array_column($episodes, 'position'), + array_map(function($episode) { + // In case you don't have active subscription. + if (! array_key_exists('download', $episode)) + return null; + + return [ + 'title' => $episode['title'], + // Some video links starts with '//' and doesn't include protocol + 'download_link' => strpos($episode['download'], 'https:') === 0 + ? $episode['download'] + : 'https:' . $episode['download'], + 'number' => $episode['position'] + ]; + }, $episodes) + ) + ); } - /** - * Extracts the name of the episode. - * - * @param $html - * - * @param $path - * @return string - */ - public static function getNameOfEpisode($html, $path) + public static function getCsrfToken($html) { - $parser = new Crawler($html); - $t = $parser->filter("h4 a[href='/".$path."']")->text(); + preg_match('/"csrfToken": \'([^\s]+)\'/', $html, $matches); - return trim($t); + return $matches[1]; } - public static function getSeriesArray($html) + public static function getUserData($html) { - $parser = new Crawler($html); - $seriesNodes = $parser->filter(".expanded-card-meta-lessons a"); + $data = self::getData($html); - $series = $seriesNodes->each(function(Crawler $crawler) { - $slug = str_replace('/series/', '', $crawler->attr('href')); - $episode_count = (int) $crawler->text(); + $props = $data['props']; - return [ - 'slug' => $slug, - 'episode_count' => $episode_count, - ]; - }); + return [ + 'error' => empty($props['errors']) ? null : $props['errors']['auth'], + 'signedIn' => $props['auth']['signedIn'], + 'data' => $props['auth']['user'] + ]; + } + + /** + * Returns decoded version of data-page attribute in HTML page + * + * @param string $html + * @return array + */ + private static function getData($html) + { + $parser = new Crawler($html); + + $data = $parser->filter("#app")->attr('data-page'); - return $series; + return json_decode($data, true); } } diff --git a/App/Http/Resolver.php b/App/Http/Resolver.php index 21a6439..f34335e 100644 --- a/App/Http/Resolver.php +++ b/App/Http/Resolver.php @@ -5,10 +5,6 @@ namespace App\Http; -use App\Downloader; -use App\Exceptions\EpisodePageNotFoundException; -use App\Exceptions\NoDownloadLinkException; -use App\Exceptions\SubscriptionNotActiveException; use App\Html\Parser; use App\Utils\Utils; use GuzzleHttp\Client; @@ -19,30 +15,35 @@ /** * Class Resolver + * * @package App\Http */ class Resolver { /** * Guzzle client + * * @var Client */ private $client; /** * Guzzle cookie + * * @var CookieJar */ private $cookie; /** * Ubench lib + * * @var Ubench */ private $bench; /** * Retry download on connection fail + * * @var int */ private $retryDownload = false; @@ -63,195 +64,146 @@ public function __construct(Client $client, Ubench $bench, $retryDownload = fals } /** - * Tries to auth. - * - * @param $email - * @param $password + * Tries to authenticate user. * - * @return bool - * @throws SubscriptionNotActiveException + * @param string $email + * @param string $password + * @return array */ - public function doAuth($email, $password) + public function login($email, $password) { - $response = $this->client->get(LARACASTS_LOGIN_PATH, [ - 'cookies' => $this->cookie, - 'verify' => false - ]); - - $token = Parser::getToken($response->getBody()->getContents()); - $response = $this->client->post(LARACASTS_POST_LOGIN_PATH, [ 'cookies' => $this->cookie, - 'body' => [ - 'email' => $email, - 'password' => $password, - '_token' => $token, - 'remember' => 1, + 'headers' => [ + "X-CSRF-TOKEN" => $this->getCsrfToken(), + 'content-type' => 'application/json', ], + 'body' => json_encode([ + 'email' => $email, + 'password' => $password, + 'remember' => 1 + ]), 'verify' => false ]); $html = $response->getBody()->getContents(); - if (strpos($html, "Reactivate") !== FALSE) { - throw new SubscriptionNotActiveException(); - } + return Parser::getUserData($html); + } - if(strpos($html, "The email must be a valid email address.") !== FALSE) { - return false; - } + /** + * Returns CSRF token + * + * @return string + */ + public function getCsrfToken() + { + $response = $this->client->get(LARACASTS_BASE_URL, [ + 'cookies' => $this->cookie, + 'verify' => false + ]); - // user doesnt provided an email in the .env - // laracasts redirects to login page again - if(strpos($html, 'name="password"') !== FALSE) { - return false; - } + $html = $response->getBody()->getContents(); - return strpos($html, "verify your credentials.") === FALSE; + return Parser::getCsrfToken($html); } /** * Download the episode of the serie. * - * @param $serie - * @param $episode + * @param string $serieSlug + * @param array $episode * @return bool */ - public function downloadSerieEpisode($serie, $episode) + public function downloadEpisode($serieSlug, $episode) { try { - $path = LARACASTS_SERIES_PATH . '/' . $serie . '/episodes/' . $episode; - $episodePage = $this->getPage($path); - $name = $this->getNameOfEpisode($episodePage, $path); - $number = sprintf("%02d", $episode); - $saveTo = BASE_FOLDER . '/' . SERIES_FOLDER . '/' . $serie . '/' . $number . '-' . $name . '.mp4'; - Utils::writeln(sprintf("Download started: %s . . . . Saving on " . SERIES_FOLDER . '/' . $serie . ' folder.', - $number . ' - ' . $name + $name = $episode['title']; + + $number = sprintf("%02d", $episode['number']); + + $saveTo = BASE_FOLDER + . DIRECTORY_SEPARATOR + . SERIES_FOLDER + . DIRECTORY_SEPARATOR + . $serieSlug + . DIRECTORY_SEPARATOR + . $number . '-' . Utils::parseEpisodeName($name) . '.mp4'; + + Utils::writeln( + sprintf( + "Download started: %s . . . . Saving on " . SERIES_FOLDER . '/' . $serieSlug, + $number . ' - ' . $name )); - return $this->downloadLessonFromPath($episodePage, $saveTo); - } catch (EpisodePageNotFoundException $e) { - Utils::write(sprintf($e->getMessage())); - return false; + return $this->downloadVideo($episode['download_link'], $saveTo); } catch (RequestException $e) { Utils::write(sprintf($e->getMessage())); + return false; } } + /** - * Downloads the lesson. + * Returns topics page html * - * @param $lesson - * @return bool + * @return string */ - public function downloadLesson($lesson) + public function getTopicsHtml() { - try - { - $path = LARACASTS_LESSONS_PATH . '/' . $lesson; - $number = sprintf("%04d", Downloader::$totalLocalLessons + Downloader::$currentLessonNumber --); - $saveTo = BASE_FOLDER . '/' . LESSONS_FOLDER . '/' . $number . '-' . $lesson . '.mp4'; - - Utils::writeln(sprintf("Download started: %s . . . . Saving on " . LESSONS_FOLDER . ' folder.', - $lesson - )); - $html = $this->getPage($path); - - return $this->downloadLessonFromPath($html, $saveTo); - } catch (RequestException $e) { - Utils::write(sprintf($e->getMessage())); - return false; - } + return $this->client + ->get(LARACASTS_BASE_URL . '/' . LARACASTS_TOPICS_PATH, ['cookies' => $this->cookie, 'verify' => false]) + ->getBody() + ->getContents(); } /** - * Helper function to get html of a page - * @param $path + * Returns html content of specific url + * + * @param string $url * @return string */ - private function getPage($path) { - $response = $this->client->get($path, [ - 'cookies' => $this->cookie, - 'verify' => false, - 'allow_redirects' => false - ]); - - if ($response->getStatusCode() == 302) { - throw new EpisodePageNotFoundException("The episode page not found at: $path"); - } - - return $response->getBody()->getContents(); + public function getHtml($url) + { + return $this->client + ->get($url, ['cookies' => $this->cookie, 'verify' => false]) + ->getBody() + ->getContents(); } /** * Helper to get the Location header. * * @param $url - * * @return string */ private function getRedirectUrl($url) { $response = $this->client->get($url, [ - 'cookies' => $this->cookie, - 'allow_redirects' => FALSE, + 'cookies' => $this->cookie, + 'allow_redirects' => false, 'verify' => false ]); return $response->getHeader('Location'); } - /** - * Gets the name of the serie episode. - * - * @param $html - * - * @param $path - * @return string - */ - private function getNameOfEpisode($html, $path) - { - $name = Parser::getNameOfEpisode($html, $path); - - return Utils::parseEpisodeName($name); - } - /** * Helper to download the video. * - * @param $html + * @param $downloadUrl * @param $saveTo * @return bool */ - private function downloadLessonFromPath($html, $saveTo) + private function downloadVideo($downloadUrl, $saveTo) { - $scheduled = Parser::scheduledEpisode($html); - if ($scheduled !== false) { - Utils::write(sprintf("This lesson is not available yet. Retry later: %s", $scheduled)); - return false; - } - - try { - $downloadUrl = Parser::getDownloadLink($html); - $viemoUrl = $this->getRedirectUrl($downloadUrl); - $finalUrl = $this->getRedirectUrl($viemoUrl); - } catch(NoDownloadLinkException $e) { - Utils::write(sprintf("Can't download this lesson! :( No download button")); - - try { - Utils::write(sprintf("Tring to find a Wistia.net video")); - $Wistia = new Wistia($html,$this->bench); - $finalUrl = $Wistia->getDownloadUrl(); - } catch(NoDownloadLinkException $e) { - return false; - } - - } - $this->bench->start(); + $finalUrl = $this->getRedirectUrl($downloadUrl); + $retries = 0; + while (true) { try { $downloadedBytes = file_exists($saveTo) ? filesize($saveTo) : 0; @@ -264,7 +216,7 @@ private function downloadLessonFromPath($html, $saveTo) ]); if (php_sapi_name() == "cli") { //on cli show progress - $req->getEmitter()->on('progress', function (ProgressEvent $e) use ($downloadedBytes) { + $req->getEmitter()->on('progress', function(ProgressEvent $e) use ($downloadedBytes) { printf("> Total: %d%% Downloaded: %s of %s \r", Utils::getPercentage($e->downloaded + $downloadedBytes, $e->downloadSize), Utils::formatBytes($e->downloaded + $downloadedBytes), @@ -272,18 +224,10 @@ private function downloadLessonFromPath($html, $saveTo) }); } - $response = $this->client->send($req); - - if(strpos($response->getHeader('Content-Type'), 'text/html') !== FALSE) { - Utils::writeln(sprintf("Got HTML instead of the video file, the subscription is probably inactive")); - throw new SubscriptionNotActiveException(); - } + $this->client->send($req); break; } catch (\Exception $e) { - if (is_a($e, SubscriptionNotActiveException::class) || !$this->retryDownload || ($this->retryDownload && $retries >= 3)) { - throw $e; - } ++$retries; Utils::writeln(sprintf("Retry download after connection fail! ")); continue; diff --git a/App/Http/Wistia.php b/App/Http/Wistia.php deleted file mode 100644 index 776d072..0000000 --- a/App/Http/Wistia.php +++ /dev/null @@ -1,118 +0,0 @@ -client = new \GuzzleHttp\Client(['base_url' => 'http://www.clipconverter.cc']); - $this->cookie = new CookieJar(); - $this->bench = $bench; - $this->html = $html; - } - /** - * Get wistia.net video url - */ - public function getDownloadUrl() - { - try { - $this->getWistiaID(); - } catch(NoWistiaIDException $e) { - Utils::write(sprintf("Can't find any wistia.net ID! :(")); - return false; - } - - $response = $this->client->post('check.php', [ - 'cookies' => $this->cookie, - 'body' => [ - 'mediaurl' => 'http://fast.wistia.net/embed/iframe/'.$this->wistiaID - ], - 'verify' => false - ]); - - $html = $response->getBody()->getContents(); - - $data = json_decode($html,true); - - if(!isset($data['url'])) - return false; - - $finalUrl = ''; - $maxSize = 0; - foreach($data['url'] as $url) { - if($url['size'] > $maxSize) { - $maxSize = $url['size']; - $finalUrl = $url['url']; - } - } - Utils::writeln(sprintf("Found video URL %s ", - $finalUrl - )); - - return $finalUrl; - } - - /** - * Get wistia.net video id - */ - protected function getWistiaID() { - if(preg_match('~wistia_async_([a-z0-9]+)\s~',$this->html,$match)) { - $this->wistiaID = trim($match[1]); - - Utils::writeln(sprintf("Found Wistia.net ID %s ", - $this->wistiaID - )); - } - else { - return false; - } - } -} diff --git a/App/Laracasts/Controller.php b/App/Laracasts/Controller.php index 2358e38..c19d3b8 100644 --- a/App/Laracasts/Controller.php +++ b/App/Laracasts/Controller.php @@ -5,76 +5,101 @@ use App\Html\Parser; -use GuzzleHttp\Client; -use GuzzleHttp\Cookie\CookieJar; +use App\Http\Resolver; +use App\Utils\SeriesCollection; +use App\Utils\Utils; class Controller { /** - * @var \GuzzleHttp\Client + * @var \App\Http\Resolver */ private $client; - /** - * @var array - */ - private $algoliaResults; /** * Controller constructor. - * @param Client $client - */ - public function __construct(Client $client) - { - $this->client = $client; - $this->cookie = new CookieJar(); - } - - /** - * Adds algolia results for use while merging * - * @param array $algoliaLessons + * @param Resolver $client */ - public function addAlgoliaResults(array $algoliaLessons) + public function __construct(Resolver $client) { - $this->algoliaResults = $algoliaLessons; + $this->client = $client; } /** - * Gets all series with scraping and merges with algolia result + * Gets all series using scraping * + * @param array $cachedData * @return array - * @throws \Exception */ - public function getAllSeries() + public function getSeries($cachedData) { - if (empty($this->algoliaResults)) throw new \Exception('algoliaResults is empty, use addAlgoliaResults() to add a result'); + $seriesCollection = new SeriesCollection($cachedData); + + $topics = Parser::getTopicsData($this->client->getTopicsHtml()); + + foreach ($topics as $topic) { + + if ($this->isTopicUpdated($seriesCollection, $topic)) + continue; + + Utils::box($topic['slug']); - $seriesHtml = $this->getSeriesHtml(); + $topicHtml = $this->client->getHtml($topic['path']); - $mergedResult = $this->algoliaResults; + $series = Parser::getSeriesData($topicHtml); - $series = Parser::getSeriesArray($seriesHtml); + foreach ($series as $serie) { + if ($this->isSerieUpdated($seriesCollection, $serie)) + continue; - foreach ($series as $serie) { - foreach (range(1, $serie['episode_count']) as $episode) { - $key = $episode - 1; // Since we override we can't just append to the array - $mergedResult['series'][$serie['slug']][$key] = $episode; + Utils::writeln("Getting serie: {$serie['slug']} ..."); + + $seriHtml = $this->client->getHtml($serie['path']); + + $serie['topic'] = $topic['slug']; + + $serie['episodes'] = Parser::getEpisodesData($seriHtml); + + $seriesCollection->add($serie); } + } - return $mergedResult; + return $seriesCollection->get(); } + /** - * Returns series page html + * Determine is specific topic has been changed compared to cached data * - * @return string + * @param SeriesCollection $series + * @param array $topic + * @return bool + * */ + public function isTopicUpdated($series, $topic) + { + $series = $series->where('topic', $topic['slug']); + + return + $series->exists() + and + $topic['series_count'] == $series->count() + and + $topic['episode_count'] == $series->sum('episode_count', true); + } + + /** + * Determine is specific series has been changed compared to cached data + * + * @param SeriesCollection $series + * @param array $serie + * @return bool */ - private function getSeriesHtml() + private function isSerieUpdated($series, $serie) { - return $this->client - ->get(LARACASTS_BASE_URL . '/' . LARACASTS_SERIES_PATH, ['cookies' => $this->cookie, 'verify' => false]) - ->getBody() - ->getContents(); + $target = $series->where('slug', $serie['slug'])->first(); + + return ! is_null($target) and (count($target['episodes']) == $serie['episode_count']); } -} \ No newline at end of file +} diff --git a/App/System/Controller.php b/App/System/Controller.php index 99362a2..9e9e3f3 100644 --- a/App/System/Controller.php +++ b/App/System/Controller.php @@ -4,7 +4,6 @@ */ namespace App\System; -use App\Downloader; use App\Utils\Utils; use League\Flysystem\Filesystem; @@ -30,22 +29,6 @@ public function __construct(Filesystem $system) $this->system = $system; } - /** - * Gets the array of the local lessons & series. - * - * @return array - */ - public function getAllLessons() - { - $array = []; - $array['lessons'] = $this->getLessons(true); - $array['series'] = $this->getSeries(true); - - Downloader::$totalLocalLessons = count($array['lessons']); - - return $array; - } - /** * Get the series * @@ -53,7 +36,7 @@ public function getAllLessons() * * @return array */ - private function getSeries($skip = false) + public function getSeries($skip = false) { $list = $this->system->listContents(SERIES_FOLDER, true); $array = []; @@ -61,7 +44,9 @@ private function getSeries($skip = false) foreach ($list as $entry) { if ($entry['type'] != 'file') { continue; - } //skip folder, we only want the files + } + + //skip folder, we only want the files if (substr($entry['filename'], 0, 2) == '._') { continue; } @@ -72,67 +57,25 @@ private function getSeries($skip = false) $array[$serie][] = $episode; } + // TODO: #Issue# returns array with index 0 if($skip) { - foreach($this->getSkipSeries() as $skipSerie => $episodes) { + foreach($this->getSkippedSeries() as $skipSerie => $episodes) { if(!isset($array[$skipSerie])) { $array[$skipSerie] = $episodes; continue; } - $array[$skipSerie] = array_merge($array[$skipSerie], $episodes); - $array[$skipSerie] = array_filter(array_unique($array[$skipSerie])); - } - } - - return $array; - } - - /** - * Gets the lessons in the folder. - * - * @param bool $skip - * - * @return array - */ - public function getLessons($skip = false) - { - $list = $this->system->listContents(LESSONS_FOLDER); - $array = []; - - foreach ($list as $entry) { - if ($entry['type'] != 'file') { - continue; + $array[$skipSerie] = array_filter( + array_unique( + array_merge($array[$skipSerie], $episodes) + ) + ); } - - $originalName = $entry['filename']; - - $array[] = substr($originalName, strpos($originalName, '-') + 1); - } - - if ($skip) { - $array = array_merge($this->getSkipLessons(), $array); - $array = array_filter(array_unique($array)); } return $array; } - /** - * Create skip file to lessons - */ - public function writeSkipLessons() - { - $file = LESSONS_FOLDER . '/.skip'; - - $lessons = serialize($this->getLessons(true)); - - if($this->system->has($file)) { - $this->system->delete($file); - } - - $this->system->write($file, $lessons); - } - /** * run write commands */ @@ -141,18 +84,14 @@ public function writeSkipFiles() Utils::box('Creating skip files'); $this->writeSkipSeries(); - Utils::write('Skip files for series created'); - - $this->writeSkipLessons(); - Utils::write('Skip files for lesson created'); - Utils::box('Finished'); + Utils::write('Skip files for series created'); } /** * Create skip file to lessons */ - public function writeSkipSeries() + private function writeSkipSeries() { $file = SERIES_FOLDER . '/.skip'; @@ -166,21 +105,12 @@ public function writeSkipSeries() } /** - * Get skiped lessons - * @return array - */ - public function getSkipLessons() - { - return $this->getSkipedData(LESSONS_FOLDER . '/.skip'); - } - - /** - * Get skiped series + * Get skipped series * @return array */ - public function getSkipSeries() + private function getSkippedSeries() { - return $this->getSkipedData(SERIES_FOLDER . '/.skip'); + return $this->getSkippedData(SERIES_FOLDER . '/.skip'); } /** @@ -189,8 +119,8 @@ public function getSkipSeries() * @param $pathToSkipFile * @return array|mixed */ - private function getSkipedData($pathToSkipFile) { - + private function getSkippedData($pathToSkipFile) + { if ($this->system->has($pathToSkipFile)) { $content = $this->system->read($pathToSkipFile); @@ -200,41 +130,14 @@ private function getSkipedData($pathToSkipFile) { return []; } - /** - * Rename lessons, adding 0 padding to the number. - */ - public function renameLessonsWithRightPadding() - { - $list = $this->system->listContents(LESSONS_FOLDER); - - foreach ($list as $entry) { - if ($entry['type'] != 'file') { - continue; - } - - $originalName = $entry['basename']; - $oldNumber = substr($originalName, 0, strpos($originalName, '-')); - - if (strlen($oldNumber) == 4) { - continue; - } // already correct - - $newNumber = sprintf("%04d", $oldNumber); - $nameWithoutNumber = substr($originalName, strpos($originalName, '-') + 1); - $newName = $newNumber . '-' . $nameWithoutNumber; - - $this->system->rename(LESSONS_FOLDER . '/' . $originalName, LESSONS_FOLDER . '/' . $newName); - } - } - /** * Create series folder if not exists. * - * @param $serie + * @param $serieSlug */ - public function createSerieFolderIfNotExists($serie) + public function createSerieFolderIfNotExists($serieSlug) { - $this->createFolderIfNotExists(SERIES_FOLDER . '/' . $serie); + $this->createFolderIfNotExists(SERIES_FOLDER . '/' . $serieSlug); } /** @@ -248,4 +151,36 @@ public function createFolderIfNotExists($folder) $this->system->createDir($folder); } } + + /** + * Create cache file + * + * @param array $data + * @throws \League\Flysystem\FileExistsException + * @throws \League\Flysystem\FileNotFoundException + */ + public function setCache($data) + { + $file = 'cache.php'; + + if ($this->system->has($file)) { + $this->system->delete($file); + } + + $this->system->write($file, 'system->has($file) + ? require $this->system->getAdapter()->getPathPrefix() . $file + : []; + } } diff --git a/App/Utils/SeriesCollection.php b/App/Utils/SeriesCollection.php new file mode 100644 index 0000000..336f0b9 --- /dev/null +++ b/App/Utils/SeriesCollection.php @@ -0,0 +1,76 @@ +series = $series; + } + + /** + * @param string $key + * @param string $value + * @return $this + */ + public function where($key, $value) + { + $series = []; + + foreach ($this->series as $serie) { + if ($serie[$key] == $value) { + array_push($series, $serie); + } + } + + return new SeriesCollection($series); + } + + public function sum($key, $actual) + { + $sum = 0; + + foreach ($this->series as $serie) { + if ($actual) { + $sum += intval(count($serie[str_replace('_count', '', $key) . 's'])); + } else { + $sum += intval($serie[$key]); + } + } + + return $sum; + } + + public function count() + { + return (int) count($this->series); + } + + public function get() + { + return $this->series; + } + + public function exists() + { + return ! empty($this->series); + } + + public function first() + { + return $this->exists() ? $this->series[0] : null; + } + + public function add($serie) + { + $this->series[$serie['slug']] = $serie; + } +} diff --git a/App/Utils/Utils.php b/App/Utils/Utils.php index 2d2afca..cc40e9d 100644 --- a/App/Utils/Utils.php +++ b/App/Utils/Utils.php @@ -24,33 +24,6 @@ public static function newLine() return "
"; } - /** - * Count the total lessons of an array of lessons & series. - * - * @param $array - * - * @return int - */ - public static function countAllLessons($array) - { - $total = count($array['lessons']); - $total += self::countEpisodes($array); - - return $total; - } - - /** - * Counts the lessons from the array. - * - * @param $array - * - * @return int - */ - public static function countLessons($array) - { - return count($array['lessons']); - } - /** * Counts the episodes from the array. * @@ -61,8 +34,9 @@ public static function countLessons($array) public static function countEpisodes($array) { $total = 0; - foreach ($array['series'] as $serie) { - $total += count($serie); + + foreach ($array as $serie) { + $total += count($serie['episodes']); } return $total; @@ -76,31 +50,33 @@ public static function countEpisodes($array) * * @return array */ - public static function resolveFaultyLessons($onlineListArray, $localListArray) + public static function compareLocalAndOnlineSeries($onlineListArray, $localListArray) { - $array = []; - $array['series'] = []; - $array['lessons'] = []; + $seriesCollection = new SeriesCollection([]); - foreach ($onlineListArray['series'] as $serie => $episodes) { - if (isset($localListArray['series'][$serie])) { - if (count($episodes) == count($localListArray['series'][$serie])) { + foreach ($onlineListArray as $serieSlug => $serie) { + + if (array_key_exists($serieSlug, $localListArray)) { + if ($serie['episode_count'] == count($localListArray[$serieSlug])) { continue; } - foreach ($episodes as $episode) { - if (!in_array($episode, $localListArray['series'][$serie])) { - $array['series'][$serie][] = $episode; + $episodes = $serie['episodes']; + $serie['episodes'] = []; + + foreach ($episodes as $number => $episode) { + if (!in_array($number, $localListArray[$serieSlug])) { + $serie['episodes'][$number] = $episode; } } + + $seriesCollection->add($serie); } else { - $array['series'][$serie] = $episodes; + $seriesCollection->add($serie); } } - $array['lessons'] = array_diff($onlineListArray['lessons'], $localListArray['lessons']); - - return $array; + return $seriesCollection->get(); } /** @@ -135,15 +111,7 @@ public static function write($text) */ public static function parseEpisodeName($name) { - $toRemove = 'New'; - $striped = preg_replace('/[^A-Za-z0-9\- _]/', '', $name); - - if (strpos($striped, $toRemove) !== false) { //remove last New string - $striped = preg_replace('/'. preg_quote($toRemove, '/') . '$/', '', $striped); - return rtrim($striped); - } - - return $striped; + return preg_replace('/[^A-Za-z0-9\- _]/', '', $name); } /** diff --git a/README.md b/README.md index 83d1c15..88c5ea8 100644 --- a/README.md +++ b/README.md @@ -8,11 +8,6 @@ Downloads new lessons and series from laracasts if there are updates. Or the who **Currently looking for maintainers.** -### LIMITED FUNCTIONALITY NOTE: -Due to recent changes in the structure of the series page, it is no longer possible to fetch the full catalog -of lessons and series. Between a lengthy (unknown) delay on algolia indexing, and incomplete list of content -on the series page, ability to download is severely hindered. - ## Description Syncs your local folder with the laracasts website, when there are new lessons the app download it for you. If your local folder is empty, all lessons and series will be downloaded! @@ -21,11 +16,17 @@ A .skip file is used to prevent downloading deleted lessons for these with space Just call `php makeskips.php` before deleting the lessons. -## An account with an active subscription is necessary! -Even to download free lessons or series. The download option is only allowed to users with a valid subscription. +## Do I need an active subscription account? +You ONLY can download free episodes as guest and need active subscription to download all series. + +I suggest starting the script as a guest at first step, this helps you to don't exceed daily download limits. +Then fill your account credentials in ``.env`` and re-run project. + +> You need active subscription account to take advantage of cache feature. + ## Requirements -- PHP >= 5.4 +- PHP >= 7.0 - php-cURL - php-xml - php-json @@ -41,13 +42,7 @@ OR ```sh $ cp .env.example .env ``` -3. Update the `.env` with your login and API information. To obtain this, do the following: - - Go to [laracasts.com and navigate to the Browse page](https://laracasts.com/search). - - Open your browsers Dev Tools and open the Network tab, then refresh the page. - - Find an XHR request to `algolia.net` and look at the request URL. - - Within the URL, find the GET parameters: - - Copy the `x-algolia-application-id` value to `ALGOLIA_APP_ID` in `.env`. - - Copy the `x-algolia-api-key` value to `ALGOLIA_API_KEY` in `.env`. +3. Update your laracasts account credentials (EMAIL, PASSWORD) in ``.env`` 4. The next steps, choose if you want a [local installation](#using-your-local-machine) or [a Docker based installation](#using-docker) and follow along. ### Using your local machine @@ -112,18 +107,6 @@ This will only download episodes which you mentioned in as usual. -### Command to download specific lessons -You can either use the Lessons slug (preferred): -```sh -$ php start.php -l "lesson-slug-example" -$ php start.php --lesson-name "lesson-slug-example" -``` -Or the Lesson name: -```sh -$ php start.php -l "Lesson name example" -$ php start.php --lesson-name "Lessons name example" -``` - ## Troubleshooting If you have a `cURL error 60: SSL certificate problem: self signed certificate in certificate chain` or `SLL error: cURL error 35` do this: diff --git a/bootstrap.php b/bootstrap.php index 30ab7a0..6ab4db2 100644 --- a/bootstrap.php +++ b/bootstrap.php @@ -27,24 +27,18 @@ $options['series_folder'] = getenv('SERIES_FOLDER'); //Flags $options['retry_download'] = boolval(getenv('RETRY_DOWNLOAD')); -//Algolia -$options['algolia_app_id'] = getenv('ALGOLIA_APP_ID'); -$options['algolia_api_key'] = getenv('ALGOLIA_API_KEY'); define('BASE_FOLDER', $options['local_path']); define('LESSONS_FOLDER', $options['lessons_folder']); define('SERIES_FOLDER', $options['series_folder']); define('RETRY_DOWNLOAD', $options['retry_download']); -define('ALGOLIA_APP_ID', $options['algolia_app_id']); -define('ALGOLIA_API_KEY', $options['algolia_api_key']); -define('ALGOLIA_INDEX_NAME', 'lessons'); + //laracasts define('LARACASTS_BASE_URL', 'https://laracasts.com'); -define('LARACASTS_LOGIN_PATH', 'login'); define('LARACASTS_POST_LOGIN_PATH', 'sessions'); -define('LARACASTS_LESSONS_PATH', 'lessons'); define('LARACASTS_SERIES_PATH', 'series'); +define('LARACASTS_TOPICS_PATH', 'browse/all'); /* * Vars diff --git a/composer.json b/composer.json index af89f83..cf2a6ad 100644 --- a/composer.json +++ b/composer.json @@ -12,8 +12,7 @@ "ext-json": "*", "ext-xml": "*", "vlucas/phpdotenv": "^2.3", - "cocur/slugify": "^2.3", - "algolia/algoliasearch-client-php": "~1.28" + "cocur/slugify": "^2.3" }, "require-dev": { "digitalnature/php-ref": "dev-master" diff --git a/composer.lock b/composer.lock index 314727d..14fc61a 100644 --- a/composer.lock +++ b/composer.lock @@ -4,64 +4,8 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "1b4cb21e9376b88347e97733b7ef7f0b", + "content-hash": "fb9c03973e3366a1ea7e180dcc35b71a", "packages": [ - { - "name": "algolia/algoliasearch-client-php", - "version": "1.x-dev", - "source": { - "type": "git", - "url": "https://github.com/algolia/algoliasearch-client-php.git", - "reference": "feb6b4d1972df42dc5b24900f32c6add5ee0c21d" - }, - "dist": { - "type": "zip", - "url": "https://api.github.com/repos/algolia/algoliasearch-client-php/zipball/feb6b4d1972df42dc5b24900f32c6add5ee0c21d", - "reference": "feb6b4d1972df42dc5b24900f32c6add5ee0c21d", - "shasum": "" - }, - "require": { - "ext-curl": "*", - "ext-mbstring": "*", - "php": "^5.3 || ^7.0" - }, - "require-dev": { - "phpunit/phpunit": "^4.8.35 || ^5.7 || ^6.4 || ^7.0", - "satooshi/php-coveralls": "^1.0" - }, - "type": "library", - "extra": { - "branch-alias": { - "dev-2.0": "2.0.x-dev" - } - }, - "autoload": { - "psr-0": { - "AlgoliaSearch": "src/" - } - }, - "notification-url": "https://packagist.org/downloads/", - "license": [ - "MIT" - ], - "authors": [ - { - "name": "Algolia Team", - "email": "contact@algolia.com" - }, - { - "name": "Ryan T. Catlin", - "email": "ryan.catlin@gmail.com" - }, - { - "name": "Jonathan H. Wage", - "email": "jonwage@gmail.com" - } - ], - "description": "Algolia Search API Client for PHP", - "homepage": "https://github.com/algolia/algoliasearch-client-php", - "time": "2018-11-21T15:49:50+00:00" - }, { "name": "cocur/slugify", "version": "2.5.x-dev", @@ -124,11 +68,15 @@ "slug", "slugify" ], + "support": { + "issues": "https://github.com/cocur/slugify/issues", + "source": "https://github.com/cocur/slugify/tree/master" + }, "time": "2017-03-23T21:52:55+00:00" }, { "name": "devster/ubench", - "version": "dev-master", + "version": "1.2.0", "source": { "type": "git", "url": "https://github.com/devster/ubench.git", @@ -169,6 +117,10 @@ "library", "micro" ], + "support": { + "issues": "https://github.com/devster/ubench/issues", + "source": "https://github.com/devster/ubench/tree/master" + }, "time": "2015-06-02T08:26:32+00:00" }, { @@ -177,12 +129,12 @@ "source": { "type": "git", "url": "https://github.com/guzzle/guzzle.git", - "reference": "d9a19b2509af0979de323707c397e87bb00ceb6b" + "reference": "d59d5293b4a804d6436d64fec71a8c75110a3165" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/guzzle/guzzle/zipball/d9a19b2509af0979de323707c397e87bb00ceb6b", - "reference": "d9a19b2509af0979de323707c397e87bb00ceb6b", + "url": "https://api.github.com/repos/guzzle/guzzle/zipball/d59d5293b4a804d6436d64fec71a8c75110a3165", + "reference": "d59d5293b4a804d6436d64fec71a8c75110a3165", "shasum": "" }, "require": { @@ -222,7 +174,11 @@ "rest", "web service" ], - "time": "2018-12-21T16:20:39+00:00" + "support": { + "issues": "https://github.com/guzzle/guzzle/issues", + "source": "https://github.com/guzzle/guzzle/tree/5.3" + }, + "time": "2020-02-18T09:14:58+00:00" }, { "name": "guzzlehttp/ringphp", @@ -273,6 +229,11 @@ } ], "description": "Provides a simple API and specification that abstracts away the details of HTTP into a single PHP function.", + "support": { + "issues": "https://github.com/guzzle/RingPHP/issues", + "source": "https://github.com/guzzle/RingPHP/tree/1.1.1" + }, + "abandoned": true, "time": "2018-07-31T13:22:33+00:00" }, { @@ -323,35 +284,40 @@ "Guzzle", "stream" ], + "support": { + "issues": "https://github.com/guzzle/streams/issues", + "source": "https://github.com/guzzle/streams/tree/master" + }, + "abandoned": true, "time": "2016-04-13T16:32:01+00:00" }, { "name": "league/flysystem", - "version": "dev-master", + "version": "1.x-dev", "source": { "type": "git", "url": "https://github.com/thephpleague/flysystem.git", - "reference": "755ba7bf3fb9031e6581d091db84d78275874396" + "reference": "c995bb0c23c58c9813d081f9523c9b7bb496698e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/755ba7bf3fb9031e6581d091db84d78275874396", - "reference": "755ba7bf3fb9031e6581d091db84d78275874396", + "url": "https://api.github.com/repos/thephpleague/flysystem/zipball/c995bb0c23c58c9813d081f9523c9b7bb496698e", + "reference": "c995bb0c23c58c9813d081f9523c9b7bb496698e", "shasum": "" }, "require": { "ext-fileinfo": "*", - "php": ">=5.5.9" + "league/mime-type-detection": "^1.3", + "php": "^7.2.5 || ^8.0" }, "conflict": { "league/flysystem-sftp": "<1.0.6" }, "require-dev": { - "phpspec/phpspec": "^3.4", - "phpunit/phpunit": "^5.7.10" + "phpspec/prophecy": "^1.11.1", + "phpunit/phpunit": "^8.5.8" }, "suggest": { - "ext-fileinfo": "Required for MimeType", "ext-ftp": "Allows you to use FTP server storage", "ext-openssl": "Allows you to use FTPS server storage", "league/flysystem-aws-s3-v2": "Allows you to use S3 storage with AWS SDK v2", @@ -407,7 +373,73 @@ "sftp", "storage" ], - "time": "2019-03-30T13:22:34+00:00" + "support": { + "issues": "https://github.com/thephpleague/flysystem/issues", + "source": "https://github.com/thephpleague/flysystem/tree/1.x" + }, + "funding": [ + { + "url": "https://offset.earth/frankdejonge", + "type": "other" + } + ], + "time": "2021-11-28T21:50:23+00:00" + }, + { + "name": "league/mime-type-detection", + "version": "1.9.0", + "source": { + "type": "git", + "url": "https://github.com/thephpleague/mime-type-detection.git", + "reference": "aa70e813a6ad3d1558fc927863d47309b4c23e69" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/thephpleague/mime-type-detection/zipball/aa70e813a6ad3d1558fc927863d47309b4c23e69", + "reference": "aa70e813a6ad3d1558fc927863d47309b4c23e69", + "shasum": "" + }, + "require": { + "ext-fileinfo": "*", + "php": "^7.2 || ^8.0" + }, + "require-dev": { + "friendsofphp/php-cs-fixer": "^3.2", + "phpstan/phpstan": "^0.12.68", + "phpunit/phpunit": "^8.5.8 || ^9.3" + }, + "type": "library", + "autoload": { + "psr-4": { + "League\\MimeTypeDetection\\": "src" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "MIT" + ], + "authors": [ + { + "name": "Frank de Jonge", + "email": "info@frankdejonge.nl" + } + ], + "description": "Mime-type detection for Flysystem", + "support": { + "issues": "https://github.com/thephpleague/mime-type-detection/issues", + "source": "https://github.com/thephpleague/mime-type-detection/tree/1.9.0" + }, + "funding": [ + { + "url": "https://github.com/frankdejonge", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/league/flysystem", + "type": "tidelift" + } + ], + "time": "2021-11-21T11:48:40+00:00" }, { "name": "react/promise", @@ -415,19 +447,19 @@ "source": { "type": "git", "url": "https://github.com/reactphp/promise.git", - "reference": "43896aa097d0233da13f81ca9df4efef5177cc82" + "reference": "29daf46dbbe351dff533cec5bc556a0ded6590be" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/reactphp/promise/zipball/43896aa097d0233da13f81ca9df4efef5177cc82", - "reference": "43896aa097d0233da13f81ca9df4efef5177cc82", + "url": "https://api.github.com/repos/reactphp/promise/zipball/29daf46dbbe351dff533cec5bc556a0ded6590be", + "reference": "29daf46dbbe351dff533cec5bc556a0ded6590be", "shasum": "" }, "require": { "php": ">=5.4.0" }, "require-dev": { - "phpunit/phpunit": "~4.8" + "phpunit/phpunit": "^9.3 || ^5.7 || ^4.8.36" }, "type": "library", "autoload": { @@ -453,7 +485,21 @@ "promise", "promises" ], - "time": "2019-01-08T16:10:34+00:00" + "support": { + "issues": "https://github.com/reactphp/promise/issues", + "source": "https://github.com/reactphp/promise/tree/2.x" + }, + "funding": [ + { + "url": "https://github.com/WyriHaximus", + "type": "github" + }, + { + "url": "https://github.com/clue", + "type": "github" + } + ], + "time": "2021-11-17T15:31:12+00:00" }, { "name": "symfony/css-selector", @@ -461,23 +507,18 @@ "source": { "type": "git", "url": "https://github.com/symfony/css-selector.git", - "reference": "8ca29297c29b64fb3a1a135e71cb25f67f9fdccf" + "reference": "da3d9da2ce0026771f5fe64cb332158f1bd2bc33" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/css-selector/zipball/8ca29297c29b64fb3a1a135e71cb25f67f9fdccf", - "reference": "8ca29297c29b64fb3a1a135e71cb25f67f9fdccf", + "url": "https://api.github.com/repos/symfony/css-selector/zipball/da3d9da2ce0026771f5fe64cb332158f1bd2bc33", + "reference": "da3d9da2ce0026771f5fe64cb332158f1bd2bc33", "shasum": "" }, "require": { "php": "^5.5.9|>=7.0.8" }, "type": "library", - "extra": { - "branch-alias": { - "dev-master": "3.4-dev" - } - }, "autoload": { "psr-4": { "Symfony\\Component\\CssSelector\\": "" @@ -491,14 +532,14 @@ "MIT" ], "authors": [ - { - "name": "Jean-François Simon", - "email": "jeanfrancois.simon@sensiolabs.com" - }, { "name": "Fabien Potencier", "email": "fabien@symfony.com" }, + { + "name": "Jean-François Simon", + "email": "jeanfrancois.simon@sensiolabs.com" + }, { "name": "Symfony Community", "homepage": "https://symfony.com/contributors" @@ -506,7 +547,24 @@ ], "description": "Symfony CssSelector Component", "homepage": "https://symfony.com", - "time": "2019-01-16T09:39:14+00:00" + "support": { + "source": "https://github.com/symfony/css-selector/tree/3.4" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2020-10-24T10:57:07+00:00" }, { "name": "symfony/dom-crawler", @@ -563,32 +621,43 @@ ], "description": "Symfony DomCrawler Component", "homepage": "https://symfony.com", + "support": { + "source": "https://github.com/symfony/dom-crawler/tree/2.8" + }, "time": "2018-11-24T22:30:19+00:00" }, { "name": "symfony/polyfill-ctype", - "version": "dev-master", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-ctype.git", - "reference": "82ebae02209c21113908c229e9883c419720738a" + "reference": "30885182c981ab175d4d034db0f6f469898070ab" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/82ebae02209c21113908c229e9883c419720738a", - "reference": "82ebae02209c21113908c229e9883c419720738a", + "url": "https://api.github.com/repos/symfony/polyfill-ctype/zipball/30885182c981ab175d4d034db0f6f469898070ab", + "reference": "30885182c981ab175d4d034db0f6f469898070ab", "shasum": "" }, "require": { - "php": ">=5.3.3" + "php": ">=7.1" + }, + "provide": { + "ext-ctype": "*" }, "suggest": { "ext-ctype": "For best performance" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11-dev" + "dev-main": "1.23-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -605,12 +674,12 @@ ], "authors": [ { - "name": "Symfony Community", - "homepage": "https://symfony.com/contributors" + "name": "Gert de Pagter", + "email": "BackEndTea@gmail.com" }, { - "name": "Gert de Pagter", - "email": "backendtea@gmail.com" + "name": "Symfony Community", + "homepage": "https://symfony.com/contributors" } ], "description": "Symfony polyfill for ctype functions", @@ -621,32 +690,57 @@ "polyfill", "portable" ], - "time": "2019-02-06T07:57:58+00:00" + "support": { + "source": "https://github.com/symfony/polyfill-ctype/tree/main" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2021-10-20T20:35:02+00:00" }, { "name": "symfony/polyfill-mbstring", - "version": "dev-master", + "version": "dev-main", "source": { "type": "git", "url": "https://github.com/symfony/polyfill-mbstring.git", - "reference": "fe5e94c604826c35a32fa832f35bd036b6799609" + "reference": "0abb51d2f102e00a4eefcf46ba7fec406d245825" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/fe5e94c604826c35a32fa832f35bd036b6799609", - "reference": "fe5e94c604826c35a32fa832f35bd036b6799609", + "url": "https://api.github.com/repos/symfony/polyfill-mbstring/zipball/0abb51d2f102e00a4eefcf46ba7fec406d245825", + "reference": "0abb51d2f102e00a4eefcf46ba7fec406d245825", "shasum": "" }, "require": { - "php": ">=5.3.3" + "php": ">=7.1" + }, + "provide": { + "ext-mbstring": "*" }, "suggest": { "ext-mbstring": "For best performance" }, + "default-branch": true, "type": "library", "extra": { "branch-alias": { - "dev-master": "1.11-dev" + "dev-main": "1.23-dev" + }, + "thanks": { + "name": "symfony/polyfill", + "url": "https://github.com/symfony/polyfill" } }, "autoload": { @@ -680,7 +774,24 @@ "portable", "shim" ], - "time": "2019-02-06T07:57:58+00:00" + "support": { + "source": "https://github.com/symfony/polyfill-mbstring/tree/main" + }, + "funding": [ + { + "url": "https://symfony.com/sponsor", + "type": "custom" + }, + { + "url": "https://github.com/fabpot", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/symfony/symfony", + "type": "tidelift" + } + ], + "time": "2021-11-30T18:21:41+00:00" }, { "name": "vlucas/phpdotenv", @@ -688,20 +799,26 @@ "source": { "type": "git", "url": "https://github.com/vlucas/phpdotenv.git", - "reference": "2a7dcf7e3e02dc5e701004e51a6f304b713107d5" + "reference": "f1e2a35e53abe9322f0ab9ada689967e30055d40" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/vlucas/phpdotenv/zipball/2a7dcf7e3e02dc5e701004e51a6f304b713107d5", - "reference": "2a7dcf7e3e02dc5e701004e51a6f304b713107d5", + "url": "https://api.github.com/repos/vlucas/phpdotenv/zipball/f1e2a35e53abe9322f0ab9ada689967e30055d40", + "reference": "f1e2a35e53abe9322f0ab9ada689967e30055d40", "shasum": "" }, "require": { - "php": ">=5.3.9", - "symfony/polyfill-ctype": "^1.9" + "php": "^5.3.9 || ^7.0 || ^8.0", + "symfony/polyfill-ctype": "^1.17" }, "require-dev": { - "phpunit/phpunit": "^4.8.35 || ^5.0" + "ext-filter": "*", + "ext-pcre": "*", + "phpunit/phpunit": "^4.8.36 || ^5.7.27 || ^6.5.14 || ^7.5.20 || ^8.5.21" + }, + "suggest": { + "ext-filter": "Required to use the boolean validator.", + "ext-pcre": "Required to use most of the library." }, "type": "library", "extra": { @@ -719,10 +836,13 @@ "BSD-3-Clause" ], "authors": [ + { + "name": "Graham Campbell", + "email": "hello@gjcampbell.co.uk" + }, { "name": "Vance Lucas", - "email": "vance@vancelucas.com", - "homepage": "http://www.vancelucas.com" + "email": "vance@vancelucas.com" } ], "description": "Loads environment variables from `.env` to `getenv()`, `$_ENV` and `$_SERVER` automagically.", @@ -731,7 +851,21 @@ "env", "environment" ], - "time": "2019-01-29T11:11:52+00:00" + "support": { + "issues": "https://github.com/vlucas/phpdotenv/issues", + "source": "https://github.com/vlucas/phpdotenv/tree/2.6" + }, + "funding": [ + { + "url": "https://github.com/GrahamCampbell", + "type": "github" + }, + { + "url": "https://tidelift.com/funding/github/packagist/vlucas/phpdotenv", + "type": "tidelift" + } + ], + "time": "2021-10-02T19:02:17+00:00" } ], "packages-dev": [ @@ -741,17 +875,18 @@ "source": { "type": "git", "url": "https://github.com/digitalnature/php-ref.git", - "reference": "da0d2c0d55798470348a2f44adb9144ee2259691" + "reference": "2381f04d7ebcbce1072e86b98a0ccbad191f3290" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/digitalnature/php-ref/zipball/da0d2c0d55798470348a2f44adb9144ee2259691", - "reference": "da0d2c0d55798470348a2f44adb9144ee2259691", + "url": "https://api.github.com/repos/digitalnature/php-ref/zipball/2381f04d7ebcbce1072e86b98a0ccbad191f3290", + "reference": "2381f04d7ebcbce1072e86b98a0ccbad191f3290", "shasum": "" }, "require": { "php": ">=5.3.0" }, + "default-branch": true, "type": "library", "autoload": { "files": [ @@ -768,7 +903,11 @@ "debug", "var_dump" ], - "time": "2019-01-16T16:45:43+00:00" + "support": { + "issues": "https://github.com/digitalnature/php-ref/issues", + "source": "https://github.com/digitalnature/php-ref/tree/v1.3" + }, + "time": "2020-01-28T21:41:16+00:00" } ], "aliases": [], @@ -785,5 +924,6 @@ "ext-json": "*", "ext-xml": "*" }, - "platform-dev": [] + "platform-dev": [], + "plugin-api-version": "2.0.0" } diff --git a/start.php b/start.php index e0ed6b0..e4cc90b 100644 --- a/start.php +++ b/start.php @@ -13,12 +13,11 @@ $client = new GuzzleHttp\Client(['base_url' => LARACASTS_BASE_URL]); $filesystem = new Filesystem(new Adapter(BASE_FOLDER)); $bench = new Ubench(); -$algolia = new AlgoliaSearch\Client(ALGOLIA_APP_ID, ALGOLIA_API_KEY); /* * App */ -$app = new App\Downloader($client, $filesystem, $bench, $algolia, RETRY_DOWNLOAD); +$app = new App\Downloader($client, $filesystem, $bench, RETRY_DOWNLOAD); try { $app->start($options);