diff --git a/.gitignore b/.gitignore index 521ef88..6a014ae 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ -*.swp +02packages.details.txt .carton -.tidyall.d/ cpanfile.snapshot local +*.swp +.tidyall.d/ diff --git a/cpanfile b/cpanfile index 3753f1b..8d18c66 100644 --- a/cpanfile +++ b/cpanfile @@ -8,6 +8,9 @@ requires 'IO::Socket::SSL'; requires 'JSON::MaybeXS'; requires 'MetaCPAN::API::Tiny'; requires 'MetaCPAN::Client', '>= 2.000000'; +requires 'Search::Elasticsearch'; +requires 'Search::Elasticsearch::Client::2_0::Direct'; +requires 'Sub::Exporter'; requires 'WWW::Mechanize::Cached'; requires 'WWW::Mechanize::Cached::GZip'; diff --git a/lib/MetaCPAN/Util.pm b/lib/MetaCPAN/Util.pm index b140451..e6905f4 100644 --- a/lib/MetaCPAN/Util.pm +++ b/lib/MetaCPAN/Util.pm @@ -3,16 +3,46 @@ package MetaCPAN::Util; use strict; use warnings; +use constant DEBUG => !!$ENV{METACPAN_EXAMPLES_DEBUG}; + use Search::Elasticsearch; use Sub::Exporter -setup => { exports => ['es'] }; sub es { return Search::Elasticsearch->new( + client => '2_0::Direct', cxn_pool => 'Static::NoPing', nodes => 'https://fastapi.metacpan.org/v1', send_get_body_as => 'POST', - trace_to => 'Stdout', + ( trace_to => 'Stdout' ) x !!(DEBUG), ); } 1; + +=head1 NAME + +MetaCPAN::Util - Utilities for accessing MetaCPAN + +=head1 DESCRIPTION + +Provides shared utility code for examples. + +=head1 FUNCTIONS + +=head2 es + +Returns a L client configured for use with the +MetaCPAN API endpoint. + +=head1 ENVIRONMENT + +=over 4 + +=item C + +When set to C<1>, enables output of verbose debugging information. + +=back + +=cut diff --git a/scripts/author/1-fetch-single-author-es.pl b/scripts/author/1-fetch-single-author-es.pl index 5d61d79..1fb7187 100755 --- a/scripts/author/1-fetch-single-author-es.pl +++ b/scripts/author/1-fetch-single-author-es.pl @@ -4,15 +4,11 @@ use warnings; use Data::Printer; -use Search::Elasticsearch; -my $es = Search::Elasticsearch->new( - cxn_pool => 'Static::NoPing', - nodes => 'fastapi.metacpan.org', - trace_to => 'Stdout', -); +use lib './lib'; +use MetaCPAN::Util qw( es ); -my $author = $es->get( +my $author = es->get( index => 'cpan', type => 'author', id => 'MSTROUT', diff --git a/scripts/author/1b-scroll-all-authors-es.pl b/scripts/author/1b-scroll-all-authors-es.pl index 85c23a0..b24b0a3 100755 --- a/scripts/author/1b-scroll-all-authors-es.pl +++ b/scripts/author/1b-scroll-all-authors-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $scroller = es()->scroll_helper( diff --git a/scripts/author/1c-scroll-all-authors-with-twitter-es.pl b/scripts/author/1c-scroll-all-authors-with-twitter-es.pl index a9d1c69..1b1e0fe 100755 --- a/scripts/author/1c-scroll-all-authors-with-twitter-es.pl +++ b/scripts/author/1c-scroll-all-authors-with-twitter-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $scroller = es()->scroll_helper( diff --git a/scripts/author/2-twitter-or-github-es.pl b/scripts/author/2-twitter-or-github-es.pl index 439a732..32b8837 100755 --- a/scripts/author/2-twitter-or-github-es.pl +++ b/scripts/author/2-twitter-or-github-es.pl @@ -4,6 +4,7 @@ use warnings; use feature qw( say ); +use lib './lib'; use MetaCPAN::Util qw( es ); my $scroller = es()->scroll_helper( diff --git a/scripts/author/twitter.pl b/scripts/author/twitter.pl new file mode 100755 index 0000000..c6a6508 --- /dev/null +++ b/scripts/author/twitter.pl @@ -0,0 +1,46 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use feature qw( say ); + +use MetaCPAN::Client; + +my $mc = MetaCPAN::Client->new( version => 'v1' ); + +# Search for authors with a listed Twitter account +my $search = $mc->author( { 'profile.name' => 'twitter' } ); + +my @handles; +while ( my $author = $search->next ) { + + # grep matching author profiles to extract only the Twitter id + my @profiles = grep { $_->{name} eq 'twitter' } @{ $author->profile }; + + foreach my $profile (@profiles) { + my $id = $profile->{id}; + + # not every handle returned by the API is prefixed by "@", so + # we'll add it when it's missing + $id = '@' . $id unless $id =~ m{\A@}; + push @handles, $id; + } +} + +# print a case-insensitive alpha-sorted list for humans to enjoy +say $_ for sort { "\L$a" cmp "\L$b" } @handles; + +=pod + +=head1 SYNOPSIS + +To create your own Twitter list of CPAN authors install and configure +L + +Then run: + + t list create cpan-authors + perl scripts/author/twitter.pl | xargs t list add cpan-authors + t list cpan-authors members + +=cut diff --git a/scripts/distribution/1-bugs-for-dist-es.pl b/scripts/distribution/1-bugs-for-dist-es.pl index 9cb0ec9..631ad5b 100755 --- a/scripts/distribution/1-bugs-for-dist-es.pl +++ b/scripts/distribution/1-bugs-for-dist-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $dist = es()->get( diff --git a/scripts/favorite/1-last-50-favorited-dists-es.pl b/scripts/favorite/1-last-50-favorited-dists-es.pl index 695eb78..5901d91 100755 --- a/scripts/favorite/1-last-50-favorited-dists-es.pl +++ b/scripts/favorite/1-last-50-favorited-dists-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $faves = es()->search( diff --git a/scripts/favorite/1a-last-100-favorited-dists-by-user-es.pl b/scripts/favorite/1a-last-100-favorited-dists-by-user-es.pl index 2d7904a..41ed3cc 100755 --- a/scripts/favorite/1a-last-100-favorited-dists-by-user-es.pl +++ b/scripts/favorite/1a-last-100-favorited-dists-by-user-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $id = shift @ARGV; @@ -17,7 +18,7 @@ query => { filtered => { query => { match_all => {} }, - filter => { term => { 'favorite.user' => $id } } + filter => { term => { 'user' => $id } } }, }, sort => [ { date => 'desc' } ], diff --git a/scripts/favorite/2-favorites-previous-month-es.pl b/scripts/favorite/2-favorites-previous-month-es.pl index ca45c40..252760f 100755 --- a/scripts/favorite/2-favorites-previous-month-es.pl +++ b/scripts/favorite/2-favorites-previous-month-es.pl @@ -5,6 +5,7 @@ use Data::Printer; use DateTime; +use lib './lib'; use MetaCPAN::Util qw( es ); my $now = DateTime->now; diff --git a/scripts/favorite/3-leaderboard-es.pl b/scripts/favorite/3-leaderboard-es.pl index a6b33b8..2b68b35 100755 --- a/scripts/favorite/3-leaderboard-es.pl +++ b/scripts/favorite/3-leaderboard-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $faves = es()->search( diff --git a/scripts/favorite/4-leaderboard-previous-month-es.pl b/scripts/favorite/4-leaderboard-previous-month-es.pl index d38e9a5..f8e1cc5 100755 --- a/scripts/favorite/4-leaderboard-previous-month-es.pl +++ b/scripts/favorite/4-leaderboard-previous-month-es.pl @@ -5,9 +5,10 @@ use Data::Printer; use DateTime; +use lib './lib'; use MetaCPAN::Util qw( es ); -my $now = DateTime->now; +my $now = DateTime->now; my $then = $now->clone->subtract( months => 1 ); my $faves = es()->search( @@ -33,6 +34,6 @@ size => 0, ); -my @dists = map { $_->{terms} } $faves->{facets}->{dist}; +my @dists = @{ $faves->{aggregations}{dist}{buckets} }; p @dists; diff --git a/scripts/favorite/5-plus-plus-your-favorites-es.pl b/scripts/favorite/5-plus-plus-your-favorites-es.pl index 26ee015..1ae1e10 100755 --- a/scripts/favorite/5-plus-plus-your-favorites-es.pl +++ b/scripts/favorite/5-plus-plus-your-favorites-es.pl @@ -6,6 +6,7 @@ use Data::Printer; use HTTP::Tiny (); use JSON::MaybeXS qw( encode_json ); +use lib './lib'; use MetaCPAN::Util qw( es ); my $token = shift @ARGV; @@ -68,7 +69,7 @@ sub plus_plus { my $params = shift; my $ua = HTTP::Tiny->new; my $res = $ua->post( - "https://api.metacpan.org/user/favorite?access_token=$token", + "https://fastapi.metacpan.org/v1/user/favorite?access_token=$token", { content => encode_json($params), headers => { 'content-type' => 'application/json' } diff --git a/scripts/favorite/6-list-plussers-by-module.pl b/scripts/favorite/6-list-plussers-by-module.pl index c30b9d5..49a4127 100755 --- a/scripts/favorite/6-list-plussers-by-module.pl +++ b/scripts/favorite/6-list-plussers-by-module.pl @@ -5,6 +5,7 @@ use feature qw( say ); use MetaCPAN::Client; +use lib './lib'; use MetaCPAN::Util qw( es ); binmode( STDOUT, ":utf8" ); diff --git a/scripts/file/1-get-files-in-dist-es.pl b/scripts/file/1-get-files-in-dist-es.pl index 30d5431..f80f37d 100755 --- a/scripts/file/1-get-files-in-dist-es.pl +++ b/scripts/file/1-get-files-in-dist-es.pl @@ -5,6 +5,7 @@ use feature qw( say ); use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $files = es()->search( diff --git a/scripts/file/2-get-dists-with-cpanfile.pl b/scripts/file/2-get-dists-with-cpanfile.pl index 5f6d7d6..224837e 100755 --- a/scripts/file/2-get-dists-with-cpanfile.pl +++ b/scripts/file/2-get-dists-with-cpanfile.pl @@ -5,6 +5,7 @@ use feature qw( say ); use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $files = es()->search( diff --git a/scripts/file/3-find-files-in-top-level-by-name.pl b/scripts/file/3-find-files-in-top-level-by-name.pl index 48d9b85..aeafec4 100755 --- a/scripts/file/3-find-files-in-top-level-by-name.pl +++ b/scripts/file/3-find-files-in-top-level-by-name.pl @@ -5,6 +5,7 @@ use feature qw( say ); use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $files = es()->search( diff --git a/scripts/file/4-main-search.pl b/scripts/file/4-main-search.pl index 80f8905..40f8c0d 100755 --- a/scripts/file/4-main-search.pl +++ b/scripts/file/4-main-search.pl @@ -5,6 +5,7 @@ use Data::Printer; use JSON::MaybeXS qw( decode_json ); +use lib './lib'; use MetaCPAN::Util qw( es ); my $search_term = shift @ARGV || 'HTML-Re'; diff --git a/scripts/module/1-fetch-single-module-es.pl b/scripts/module/1-fetch-single-module-es.pl index 633ccbe..6698c11 100755 --- a/scripts/module/1-fetch-single-module-es.pl +++ b/scripts/module/1-fetch-single-module-es.pl @@ -4,6 +4,7 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $module = es()->get( diff --git a/scripts/permission/aggregate-author-perms.pl b/scripts/permission/aggregate-author-perms.pl new file mode 100755 index 0000000..e8decbb --- /dev/null +++ b/scripts/permission/aggregate-author-perms.pl @@ -0,0 +1,178 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use feature qw( say ); + +use CLDR::Number (); +use Cpanel::JSON::XS qw( decode_json ); +use Data::Printer; +use List::Compare (); +use LWP::UserAgent (); +use Math::Round qw( nearest ); +use Parse::CPAN::Packages::Fast (); +use WWW::Mechanize::Cached (); + +my $cldr = CLDR::Number->new( locale => 'en' ); +my $decf = $cldr->decimal_formatter; +my $perf = $cldr->percent_formatter( minimum_fraction_digits => 2 ); + +my @maxmind_authors = ( + 'OALDERS', 'EILARA', 'TJMATHER', 'MATEU', 'OSCHWALD', 'RSRCHBOY', + 'FLORA', 'MARKF', 'RUBEN', 'WDS', 'KLPTWO', 'PCRONIN', + 'ANDYJACK', 'MAXMIND', +); +my @maxmind_redacted_authors = ( + 'OALDERS', 'EILARA', 'TJMATHER', 'MATEU', 'OSCHWALD', 'RSRCHBOY', + 'MARKF', 'RUBEN', 'WDS', 'KLPTWO', 'PCRONIN', 'ANDYJACK', +); + +my %PTS = ( + 'qa2010' => [ + 'CHORNY', 'HMBRAND', 'RURBAN', 'MARCEL', + 'BOOK', 'BDFOY', 'ANDYA', 'POTYL', + 'DAXIM', 'ABELTJE', 'OVID', 'HORNBURG', + 'DOMM', 'MIYAGAWA', 'PJCJ', 'BARBIE', + 'FLORA', 'RJBS', 'JKUTEJ', 'SREZIC', + 'MSCHWERN', 'SZABGAB', 'RGIERSIG', 'SQUEEK', + 'PEPL' + ], + 'qa2011' => [ + 'RGE', 'HMBRAND', 'MARCEL', 'SCHWIGON', + 'LEONT', 'BOOK', 'BDFOY', 'ANDYA', + 'AVAR', 'POTYL', 'DAXIM', 'ABELTJE', + 'SMUELLER', 'OVID', 'DOLMEN', 'WESJDJ', + 'SAPER', 'ADAMK', 'PJCJ', 'FLORA', + 'ABIGAIL', 'RJBS', 'PERLER', 'DGL', + 'JKUTEJ', 'WONKO', 'ADIE', 'BURAK' + ], + 'qa2012' => [ + 'XAV', 'HMBRAND', 'CHESSKIT', 'NPEREZ', + 'RGARCIA', 'ELIZABETH', 'DAGOLDEN', 'SCHWIGON', + 'LEONT', 'BOOK', 'BDFOY', 'ANDYA', + 'HAGGAI', 'GETTY', 'DAXIM', 'DDUMONT', + 'ABELTJE', 'OVID', 'ELBEHO', 'WOLFSAGE', + 'DOLMEN', 'WESJDJ', 'PAUAMMA', 'SAPER', + 'OALDERS', 'MIYAGAWA', 'PJCJ', 'BARBIE', + 'FLORA', 'VPIT', 'RJBS', 'DGL', + 'ISHIGAKI', 'WONKO', 'SREZIC', 'MSCHWERN', + 'GARU', 'RIBASUSHI', 'ADIE', 'APEIRON', + 'ANDK' + ], + 'qa2013' => [ + 'PERRETTDL', 'SJN', 'TBSLIVER', 'JROBINSON', + 'MITHALDU', 'HMBRAND', 'ELIZABETH', 'DAGOLDEN', + 'SCHWIGON', 'JKEENAN', 'LEONT', 'BOOK', + 'BANNAN', 'ANDYA', 'DREBOLO', 'PDCAWLEY', + 'GETTY', 'BINGOS', 'ABELTJE', 'REHSACK', + 'BYTEROCK', 'WOLFSAGE', 'DOLMEN', 'BBUSS', + 'PJCJ', 'BARBIE', 'ARC', 'RJBS', + 'DGL', 'ISHIGAKI', 'RIBASUSHI', 'JMASTROS', + 'ANDK', 'NEWELLC' + ], + 'qa2014' => [ + 'SJN', 'ETHER', 'MITHALDU', 'HMBRAND', + 'ELIZABETH', 'DAGOLDEN', 'SCHWIGON', 'LEONT', + 'BOOK', 'DAMS', 'MSTROUT', 'ABELTJE', + 'TIMB', 'FROGGS', 'OVID', 'REHSACK', + 'ELBEHO', 'WOLFSAGE', 'DOLMEN', 'SAPER', + 'OALDERS', 'PJCJ', 'BARBIE', 'RJBS', + 'ISHIGAKI', 'NEILB', 'SREZIC', 'RIBASUSHI', + 'HAARG', 'ANDK' + ], + 'qa2015' => [ + 'SJN', 'ETHER', 'MITHALDU', 'HMBRAND', + 'ELIZABETH', 'DAGOLDEN', 'SCHWIGON', 'LEONT', + 'EXODIST', 'BOOK', 'TADZIK', 'TINITA', + 'ABELTJE', 'INGY', 'FROGGS', 'REHSACK', + 'WOLFSAGE', 'DOLMEN', 'OALDERS', 'WOLLMERS', + 'MIYAGAWA', 'PJCJ', 'BARBIE', 'BARTOLIN', + 'RJBS', 'ISHIGAKI', 'NEILB', 'SREZIC', + 'DRTECH', 'RIBASUSHI', 'LICHTKIND', 'ARISTOTLE', + 'ANDK', 'NINE' + ], + 'qa2016' => [ + 'ETHER', 'HMBRAND', 'LLAP', 'ELIZABETH', + 'SCHWIGON', 'JKEENAN', 'LEONT', 'EXODIST', + 'BOOK', 'TADZIK', 'MICKEY', 'BINGOS', + 'ABELTJE', 'TIMB', 'JBERGER', 'WOLFSAGE', + 'DOLMEN', 'OALDERS', 'PJCJ', 'BARBIE', + 'ARC', 'RJBS', 'ISHIGAKI', 'XSAWYERX', + 'NEILB', 'SREZIC', 'ARISTOTLE', 'SARGIE', + 'ANDK' + ], + 'qa2017' => [ + 'ETHER', 'MITHALDU', 'HMBRAND', 'LEEJO', + 'LLAP', 'SKAJI', 'ELIZABETH', 'LEONT', + 'UGEXE', 'BOOK', 'TODDR', 'PREACTION', + 'TADZIK', 'MICKEY', 'BINGOS', 'TINITA', + 'ABELTJE', 'INGY', 'JBERGER', 'ELBEHO', + 'WOLFSAGE', 'OALDERS', 'MIYAGAWA', 'PJCJ', + 'ARC', 'ETJ', 'ISHIGAKI', 'XSAWYERX', + 'NEILB', 'SREZIC', 'GARU', 'ATOOMIC', + 'ARISTOTLE', 'HAARG', 'ANDK', 'NINE' + ], +); + +my $ua = LWP::UserAgent->new; +$ua->mirror( 'https://cpan.metacpan.org/modules/02packages.details.txt', + '02packages.details.txt' ); + +my $parser = Parse::CPAN::Packages::Fast->new('02packages.details.txt'); + +say join '|', + ( + q{}, + q{}, + 'Modules with maint', + 'Modules in 02packages', + '% of modules in 02packages', + ); +say join '---', ( ('|') x 5 ); + +for my $author ( sort @{$PTS{qa2017}} ) { + crunch_numbers( $author, [$author]); +} + +for my $group_name ( sort keys %PTS ) { + crunch_numbers( $group_name, $PTS{$group_name} ); +} + +sub crunch_numbers { + my $title = shift; + my $authors = shift; + + # The modules which these authors have release permissions on. + my %perms = get_permissions($authors); + + my $lc = List::Compare->new( [ $parser->packages ], [ keys %perms ] ); + + # The permissioned modules which actually appear in 02packages. + my @covered = $lc->get_intersection; + + my $percent + = nearest( 0.0001, scalar @covered / ( scalar $parser->packages ) ); + + say join '|', $title, $decf->format( scalar keys %perms ), + $decf->format( scalar @covered ), $perf->format($percent); +} + +sub get_permissions { + my $authors = shift; + my $mech = WWW::Mechanize::Cached->new; + + my %modules; + + my $base_url = 'http://fastapi.metacpan.org/v1/permission/by_author'; + foreach my $author ( @{$authors} ) { + my $url = "$base_url/$author"; + $mech->get($url); + my $perms = decode_json( $mech->content ); + + foreach my $perm ( @{ $perms->{permissions} } ) { + push @{ $modules{ $perm->{module_name} } }, $author; + } + } + + return %modules; +} diff --git a/scripts/release/1-pkg2url-es.pl b/scripts/release/1-pkg2url-es.pl index f649693..9d5f94a 100755 --- a/scripts/release/1-pkg2url-es.pl +++ b/scripts/release/1-pkg2url-es.pl @@ -4,15 +4,10 @@ use warnings; use feature qw( say ); -use Search::Elasticsearch; +use lib './lib'; +use MetaCPAN::Util qw( es ); -my $es = Search::Elasticsearch->new( - cxn_pool => 'Static::NoPing', - nodes => 'https://fastapi.metacpan.org/', - trace_to => 'Stdout', -); - -my $release = $es->search( +my $release = es->search( index => 'cpan', type => 'release', body => { diff --git a/scripts/release/10-all-latest-releases.pl b/scripts/release/10-all-latest-releases.pl new file mode 100755 index 0000000..db8334b --- /dev/null +++ b/scripts/release/10-all-latest-releases.pl @@ -0,0 +1,12 @@ +use strict; +use warnings; +use feature qw( say ); + +use MetaCPAN::Client (); + +my $mc = MetaCPAN::Client->new; +my $release_results = $mc->release( { status => 'latest' } ); + +while ( my $release = $release_results->next ) { + say $release->download_url; +} diff --git a/scripts/release/11-all-latest-releases-by-NEILB.pl b/scripts/release/11-all-latest-releases-by-NEILB.pl new file mode 100755 index 0000000..851663d --- /dev/null +++ b/scripts/release/11-all-latest-releases-by-NEILB.pl @@ -0,0 +1,14 @@ +use strict; +use warnings; +use feature qw( say ); + +use MetaCPAN::Client (); + +my $mc = MetaCPAN::Client->new; +my $release_results + = $mc->release( + { all => [ { author => 'NEILB', }, { status => 'latest' } ] } ); + +while ( my $release = $release_results->next ) { + say $release->download_url; +} diff --git a/scripts/release/12-all-latest-releases-by-NEILB-with-git-repository.pl b/scripts/release/12-all-latest-releases-by-NEILB-with-git-repository.pl new file mode 100755 index 0000000..79973f3 --- /dev/null +++ b/scripts/release/12-all-latest-releases-by-NEILB-with-git-repository.pl @@ -0,0 +1,20 @@ +use strict; +use warnings; +use feature qw( say ); + +use MetaCPAN::Client (); + +my $mc = MetaCPAN::Client->new; +my $release_results = $mc->release( + { + all => [ + { author => 'NEILB', }, + { status => 'latest' }, + { 'resources.repository.type' => 'git' } + ] + } +); + +while ( my $release = $release_results->next ) { + say $release->resources->{repository}->{url}; +} diff --git a/scripts/release/13-all-releases-in-last-24-hours.pl b/scripts/release/13-all-releases-in-last-24-hours.pl new file mode 100644 index 0000000..7792f11 --- /dev/null +++ b/scripts/release/13-all-releases-in-last-24-hours.pl @@ -0,0 +1,24 @@ +use strict; +use warnings; +use feature qw( say ); + +use DateTime (); +use MetaCPAN::Client (); + +my $now = DateTime->now; +my $then = DateTime->now->subtract( days => 1 ); + +my $mc = MetaCPAN::Client->new; + +my $release_set = $mc->all( + 'releases', + { + es_filter => { + range => { date => { from => $then->datetime } }, + }, + } +); + +while ( my $release = $release_set->next ) { + say $release->download_url; +} diff --git a/scripts/release/1a-module2url-es.pl b/scripts/release/1a-module2url-es.pl index 9767f0b..9ffd498 100755 --- a/scripts/release/1a-module2url-es.pl +++ b/scripts/release/1a-module2url-es.pl @@ -6,13 +6,10 @@ use Search::Elasticsearch; -my $es = Search::Elasticsearch->new( - cxn_pool => 'Static::NoPing', - nodes => 'https://fastapi.metacpan.org/', - trace_to => 'Stdout', -); +use lib './lib'; +use MetaCPAN::Util qw( es ); -my $module = $es->search( +my $module = es->search( index => 'cpan', type => 'file', body => { @@ -29,7 +26,7 @@ my $release_name = $module->{hits}{hits}[0]{_source}{release}; -my $release = $es->search( +my $release = es->search( index => 'cpan', type => 'release', body => { diff --git a/scripts/release/2-author-upload-leaderboard-es.pl b/scripts/release/2-author-upload-leaderboard-es.pl index 4f538eb..c71bd65 100755 --- a/scripts/release/2-author-upload-leaderboard-es.pl +++ b/scripts/release/2-author-upload-leaderboard-es.pl @@ -4,6 +4,8 @@ use warnings; use Data::Printer; + +use lib './lib'; use MetaCPAN::Util qw( es ); my $uploads = es()->search( diff --git a/scripts/release/3-author-uploads-one-author-es.pl b/scripts/release/3-author-uploads-one-author-es.pl index a7c93eb..062a860 100755 --- a/scripts/release/3-author-uploads-one-author-es.pl +++ b/scripts/release/3-author-uploads-one-author-es.pl @@ -4,6 +4,7 @@ use warnings; use feature qw( say ); +use lib './lib'; use MetaCPAN::Util qw( es ); my $uploads = es()->search( diff --git a/scripts/release/4-latest-release-versions-es.pl b/scripts/release/4-latest-release-versions-es.pl index 67d74df..4ac4f55 100755 --- a/scripts/release/4-latest-release-versions-es.pl +++ b/scripts/release/4-latest-release-versions-es.pl @@ -4,6 +4,8 @@ use warnings; use Data::Printer; + +use lib './lib'; use MetaCPAN::Util qw( es ); my $latest = es()->search( @@ -17,10 +19,10 @@ query => { match_all => {} }, filter => { and => [ - { term => { 'release.status' => 'latest' } }, + { term => { 'status' => 'latest' } }, { terms => { - 'release.distribution' => + 'distribution' => [ 'Moose', 'MetaCPAN-API', 'DBIx-Class' ] }, }, diff --git a/scripts/release/4a-latest-release-versions-bool-filter-es.pl b/scripts/release/4a-latest-release-versions-bool-filter-es.pl index 6fef45d..fbfcc2f 100755 --- a/scripts/release/4a-latest-release-versions-bool-filter-es.pl +++ b/scripts/release/4a-latest-release-versions-bool-filter-es.pl @@ -4,13 +4,14 @@ use warnings; use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my $latest = es()->search( index => 'cpan', type => 'release', fields => [ 'distribution', 'version' ], - size => 3, + size => 4, body => { query => { filtered => { diff --git a/scripts/release/5-latest-releases-by-author-es.pl b/scripts/release/5-latest-releases-by-author-es.pl index e9d7916..9f6dbff 100755 --- a/scripts/release/5-latest-releases-by-author-es.pl +++ b/scripts/release/5-latest-releases-by-author-es.pl @@ -4,6 +4,8 @@ use warnings; use Data::Printer; + +use lib './lib'; use MetaCPAN::Util qw( es ); my $author = shift @ARGV; diff --git a/scripts/release/6-latest-releases-with-git-repo-es.pl b/scripts/release/6-latest-releases-with-git-repo-es.pl index 63638ed..42b7bea 100755 --- a/scripts/release/6-latest-releases-with-git-repo-es.pl +++ b/scripts/release/6-latest-releases-with-git-repo-es.pl @@ -5,6 +5,7 @@ use feature qw( say ); use Data::Printer; +use lib './lib'; use MetaCPAN::Util qw( es ); my @must = ( @@ -16,12 +17,10 @@ my $scroller = es()->scroll_helper( body => { query => { - filtered => { - filter => { bool => { must => \@must } }, - }, + bool => { must => \@must }, }, }, - fields => [ 'author', 'date', 'distribution', 'name', 'resources' ], + _source => [ 'author', 'date', 'distribution', 'name', 'resources' ], search_type => 'scan', scroll => '5m', index => 'cpan', diff --git a/scripts/release/7-all-releases-es.pl b/scripts/release/7-all-releases-es.pl new file mode 100755 index 0000000..80ecb56 --- /dev/null +++ b/scripts/release/7-all-releases-es.pl @@ -0,0 +1,24 @@ +#!/usr/bin/env perl + +use strict; +use warnings; +use feature qw( say ); + +use Data::Printer; + +use lib './lib'; +use MetaCPAN::Util qw( es ); + +my $scroller = es()->scroll_helper( + search_type => 'scan', + scroll => '5m', + index => 'cpan', + type => 'release', + size => 1_000, + body => { fields => ['download_url'] }, +); + +my @urls; +while ( my $result = $scroller->next ) { + push @urls, $result->{fields}->{download_url}; +} diff --git a/scripts/release/7-all-releases.pl b/scripts/release/7-all-releases.pl index 688cfc3..c00dbfd 100755 --- a/scripts/release/7-all-releases.pl +++ b/scripts/release/7-all-releases.pl @@ -1,22 +1,12 @@ -#!/usr/bin/env perl - use strict; use warnings; use feature qw( say ); -use Data::Printer; -use MetaCPAN::Util qw( es ); +use MetaCPAN::Client (); -my $scroller = es()->scroll_helper( - search_type => 'scan', - scroll => '5m', - index => 'cpan', - type => 'release', - size => 1_000, - body => { fields => ['download_url'] }, -); +my $mc = MetaCPAN::Client->new; +my $release_results = $mc->all('releases'); -my @urls; -while ( my $result = $scroller->next ) { - push @urls, $result->{fields}->{download_url}; +while ( my $release = $release_results->next ) { + say $release->download_url; } diff --git a/scripts/release/all-releases-by-author.pl b/scripts/release/8-all-releases-by-author-es.pl similarity index 96% rename from scripts/release/all-releases-by-author.pl rename to scripts/release/8-all-releases-by-author-es.pl index 32a5e42..ede3d9a 100755 --- a/scripts/release/all-releases-by-author.pl +++ b/scripts/release/8-all-releases-by-author-es.pl @@ -4,6 +4,7 @@ use warnings; use feature qw( say ); +use lib './lib'; use MetaCPAN::Util qw( es ); my $uploads = es()->search( diff --git a/scripts/release/9-all-releases-excluding-backpan.pl b/scripts/release/9-all-releases-excluding-backpan.pl new file mode 100755 index 0000000..4418ae6 --- /dev/null +++ b/scripts/release/9-all-releases-excluding-backpan.pl @@ -0,0 +1,12 @@ +use strict; +use warnings; +use feature qw( say ); + +use MetaCPAN::Client (); + +my $mc = MetaCPAN::Client->new; +my $release_results = $mc->release( { not => { status => 'backpan' } } ); + +while ( my $release = $release_results->next ) { + say $release->download_url; +} diff --git a/scripts/search/reverse_dependencies/by-dist-www-mech.pl b/scripts/reverse_dependencies/by-dist-www-mech.pl similarity index 62% rename from scripts/search/reverse_dependencies/by-dist-www-mech.pl rename to scripts/reverse_dependencies/by-dist-www-mech.pl index c04967b..c1196d8 100755 --- a/scripts/search/reverse_dependencies/by-dist-www-mech.pl +++ b/scripts/reverse_dependencies/by-dist-www-mech.pl @@ -10,10 +10,10 @@ my $mech = WWW::Mechanize::GZip->new; $mech->get( - "https://fastapi.metacpan.org/v1/search/reverse_dependencies/carton"); + "https://fastapi.metacpan.org/v1/reverse_dependencies/dist/Carton"); my $results = decode_json( $mech->content ); my @dists - = map { $_->{_source}->{distribution} } @{ $results->{hits}->{hits} }; + = map { $_->{distribution} } @{ $results->{data} }; p @dists; diff --git a/scripts/reverse_dependencies/by-module-www-mech.pl b/scripts/reverse_dependencies/by-module-www-mech.pl new file mode 100755 index 0000000..c13dca1 --- /dev/null +++ b/scripts/reverse_dependencies/by-module-www-mech.pl @@ -0,0 +1,20 @@ +#!/usr/bin/env perl + +use strict; +use warnings; + +use Data::Printer; +use JSON::MaybeXS qw( decode_json ); +use WWW::Mechanize::GZip (); + +my $mech = WWW::Mechanize::GZip->new; + +$mech->get( + "https://fastapi.metacpan.org/v1/reverse_dependencies/module/WWW::Mechanize" +); + +my $results = decode_json( $mech->content ); + +my @dists + = map { $_->{distribution} } @{ $results->{data} }; +p @dists; diff --git a/scripts/search/reverse_dependencies/by-release-version-mech.pl b/scripts/search/reverse_dependencies/by-release-version-mech.pl deleted file mode 100755 index 2994064..0000000 --- a/scripts/search/reverse_dependencies/by-release-version-mech.pl +++ /dev/null @@ -1,20 +0,0 @@ -#!/usr/bin/env perl - -use strict; -use warnings; - -use Data::Printer; -use JSON::MaybeXS qw( decode_json ); -use WWW::Mechanize (); - -my $mech = WWW::Mechanize->new; - -$mech->get( - 'https://fastapi.metacpan.org/v1/search/reverse_dependencies/MIYAGAWA/carton-v0.9.13' -); - -my $results = decode_json( $mech->content ); - -my @dists - = map { $_->{_source}->{distribution} } @{ $results->{hits}->{hits} }; -p @dists;