From 006a32863d0b553a251dc01a2003badb45f2d8db Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 21 Apr 2018 15:05:38 +0100 Subject: [PATCH 001/892] Added river data per distribution(s) endpoints --- lib/MetaCPAN/Document/Distribution/Set.pm | 26 +++++++++ lib/MetaCPAN/Document/Distribution/Set.pm~ | 26 +++++++++ lib/MetaCPAN/Query/Distribution.pm | 57 +++++++++++++++++++ .../Server/Controller/Distribution.pm | 11 ++++ 4 files changed, 120 insertions(+) create mode 100644 lib/MetaCPAN/Document/Distribution/Set.pm create mode 100644 lib/MetaCPAN/Document/Distribution/Set.pm~ create mode 100644 lib/MetaCPAN/Query/Distribution.pm diff --git a/lib/MetaCPAN/Document/Distribution/Set.pm b/lib/MetaCPAN/Document/Distribution/Set.pm new file mode 100644 index 000000000..cf275c749 --- /dev/null +++ b/lib/MetaCPAN/Document/Distribution/Set.pm @@ -0,0 +1,26 @@ +package MetaCPAN::Document::Distribution::Set; + +use Moose; + +use MetaCPAN::Query::Distribution; + +extends 'ElasticSearchX::Model::Document::Set'; + +has query_distribution => ( + is => 'ro', + isa => 'MetaCPAN::Query::Distribution', + lazy => 1, + builder => '_build_query_distribution', + handles => [qw< get_river_data_by_dist get_river_data_by_dists >], +); + +sub _build_query_distribution { + my $self = shift; + return MetaCPAN::Query::Distribution->new( + es => $self->es, + index_name => 'cpan', + ); +} + +__PACKAGE__->meta->make_immutable; +1; diff --git a/lib/MetaCPAN/Document/Distribution/Set.pm~ b/lib/MetaCPAN/Document/Distribution/Set.pm~ new file mode 100644 index 000000000..780299753 --- /dev/null +++ b/lib/MetaCPAN/Document/Distribution/Set.pm~ @@ -0,0 +1,26 @@ +package MetaCPAN::Document::Distribution::Set; + +use Moose; + +use MetaCPAN::Query::Distribution; + +extends 'ElasticSearchX::Model::Document::Set'; + +has query_distribution => ( + is => 'ro', + isa => 'MetaCPAN::Query::Distribution', + lazy => 1, + builder => '_build_query_distribution', + handles => [qw< get_river_data_by_dist >], +); + +sub _build_query_distribution { + my $self = shift; + return MetaCPAN::Query::Distribution->new( + es => $self->es, + index_name => 'cpan', + ); +} + +__PACKAGE__->meta->make_immutable; +1; diff --git a/lib/MetaCPAN/Query/Distribution.pm b/lib/MetaCPAN/Query/Distribution.pm new file mode 100644 index 000000000..241bb1f6e --- /dev/null +++ b/lib/MetaCPAN/Query/Distribution.pm @@ -0,0 +1,57 @@ +package MetaCPAN::Query::Distribution; + +use MetaCPAN::Moose; + +with 'MetaCPAN::Query::Role::Common'; + +sub get_river_data_by_dist { + my ( $self, $dist ) = @_; + + my $query = +{ + bool => { + must => [ { term => { name => $dist } }, ] + } + }; + + my $res = $self->es->search( + index => $self->index_name, + type => 'distribution', + body => { + query => $query, + size => 999, + } + ); + $res->{hits}{total} or return {}; + + return +{ river => +{ $dist => $res->{hits}{hits}[0]{_source}{river} } }; +} + +sub get_river_data_by_dists { + my ( $self, $dist ) = @_; + + my $query = +{ + bool => { + must => [ { terms => { name => $dist } }, ] + } + }; + + my $res = $self->es->search( + index => $self->index_name, + type => 'distribution', + body => { + query => $query, + size => 999, + } + ); + $res->{hits}{total} or return {}; + + return +{ + river => +{ + map { $_->{_source}{name} => $_->{_source}{river} } + @{ $res->{hits}{hits} } + }, + }; +} + +__PACKAGE__->meta->make_immutable; +1; diff --git a/lib/MetaCPAN/Server/Controller/Distribution.pm b/lib/MetaCPAN/Server/Controller/Distribution.pm index 719587b68..a47a66ec1 100644 --- a/lib/MetaCPAN/Server/Controller/Distribution.pm +++ b/lib/MetaCPAN/Server/Controller/Distribution.pm @@ -10,5 +10,16 @@ BEGIN { extends 'MetaCPAN::Server::Controller' } with 'MetaCPAN::Server::Role::JSONP'; +sub river_data_by_dist : Path('river') : Args(1) { + my ( $self, $c, $dist ) = @_; + $c->stash_or_detach( $self->model($c)->get_river_data_by_dist($dist) ); +} + +sub river_data_by_dists : Path('river') : Args(0) { + my ( $self, $c ) = @_; + $c->stash_or_detach( $self->model($c) + ->get_river_data_by_dists( $c->read_param('distribution') ) ); +} + __PACKAGE__->meta->make_immutable; 1; From 14df22ef9522a93281c98555596ba3b0941efe17 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Thu, 26 Apr 2018 06:27:50 +0100 Subject: [PATCH 002/892] removed editor cache file from repo --- lib/MetaCPAN/Document/Distribution/Set.pm~ | 26 ---------------------- 1 file changed, 26 deletions(-) delete mode 100644 lib/MetaCPAN/Document/Distribution/Set.pm~ diff --git a/lib/MetaCPAN/Document/Distribution/Set.pm~ b/lib/MetaCPAN/Document/Distribution/Set.pm~ deleted file mode 100644 index 780299753..000000000 --- a/lib/MetaCPAN/Document/Distribution/Set.pm~ +++ /dev/null @@ -1,26 +0,0 @@ -package MetaCPAN::Document::Distribution::Set; - -use Moose; - -use MetaCPAN::Query::Distribution; - -extends 'ElasticSearchX::Model::Document::Set'; - -has query_distribution => ( - is => 'ro', - isa => 'MetaCPAN::Query::Distribution', - lazy => 1, - builder => '_build_query_distribution', - handles => [qw< get_river_data_by_dist >], -); - -sub _build_query_distribution { - my $self = shift; - return MetaCPAN::Query::Distribution->new( - es => $self->es, - index_name => 'cpan', - ); -} - -__PACKAGE__->meta->make_immutable; -1; From 83ad589b46c6c0dd2de19192a768e7359d0a02cc Mon Sep 17 00:00:00 2001 From: "Zak B. Elep" Date: Sun, 22 Apr 2018 11:17:33 +0800 Subject: [PATCH 003/892] Add some more interesting files pertaining on how to contribute `interesting_files` already picks CONTRIBUTING and variants of it, so add a few more plus some HACKING files (which seems more prevalent in older dists.) Remove `interesting_files` filter to only root-level files This lets the function pick up Contributing.pod/Hacking.pod which are in `lib/` rather than just at the top-level. Add *.pm variants of Contributing/Hacking files Some dists like Test2-Suite use Contributing.pm instead of pod. --- lib/MetaCPAN/Query/File.pm | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index 5aeca26e6..ae16887ac 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -63,7 +63,6 @@ sub interesting_files { { bool => { must => [ - { term => { level => 0 } }, { terms => { name => [ @@ -74,6 +73,9 @@ sub interesting_files { CHANGES CONTRIBUTING CONTRIBUTING.md + CONTRIBUTING.pod + Contributing.pm + Contributing.pod COPYING COPYRIGHT CREDITS @@ -82,6 +84,12 @@ sub interesting_files { Changes Copying FAQ + HACKING + HACKING.md + HACKING.pod + Hacking.pm + Hacking.pod + Hacking INSTALL INSTALL.md LICENCE From 8848e7f98c2b8fa546cc755661971d9efa3b0b61 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Wed, 23 May 2018 10:55:46 +0200 Subject: [PATCH 004/892] -x pm file --- lib/MetaCPAN/Server.pm | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 lib/MetaCPAN/Server.pm diff --git a/lib/MetaCPAN/Server.pm b/lib/MetaCPAN/Server.pm old mode 100755 new mode 100644 From 14624a74c4ce361f0fa2b2385c330a45b9470519 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Tue, 22 May 2018 13:36:22 +0200 Subject: [PATCH 005/892] improve choices for interesting files Rather than allow any files to match by its filename alone, split the list in two. One list of files that must be in the root directory (mostly files with no extension, markdown files, or ALL CAPS files). For pod or pm files, allow them to be in subdirectories. Eliminate a few top level directories that are commonly used for special cases and not part of the dist metadata or module list. --- lib/MetaCPAN/Query/File.pm | 79 ++++++++++++++++++++++++++++++-------- 1 file changed, 63 insertions(+), 16 deletions(-) diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index ae16887ac..3fa86cba5 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -55,49 +55,61 @@ sub interesting_files { { term => { release => $release } }, { term => { author => $author } }, { term => { directory => \0 } }, - { not => { prefix => { 'path' => 'xt/' } } }, + { not => { prefix => { 'path' => 'corpus/' } } }, + { not => { prefix => { 'path' => 'fatlib/' } } }, + { not => { prefix => { 'path' => 'inc/' } } }, + { not => { prefix => { 'path' => 'local/' } } }, + { not => { prefix => { 'path' => 'perl5/' } } }, + { not => { prefix => { 'path' => 'share/' } } }, { not => { prefix => { 'path' => 't/' } } }, + { not => { prefix => { 'path' => 'xt/' } } }, { bool => { should => [ { bool => { must => [ + { term => { level => 0 } }, { terms => { name => [ qw( + alienfile AUTHORS Build.PL CHANGELOG + ChangeLog + Changelog CHANGES + Changes CONTRIBUTING CONTRIBUTING.md - CONTRIBUTING.pod - Contributing.pm - Contributing.pod + Contributing COPYING + Copying COPYRIGHT + cpanfile CREDITS - ChangeLog - Changelog - Changes - Copying + DEVELOPMENT + DEVELOPMENT.md + Development + Development.md + dist.ini FAQ + FAQ.md HACKING HACKING.md - HACKING.pod - Hacking.pm - Hacking.pod Hacking + Hacking.md INSTALL INSTALL.md LICENCE LICENSE MANIFEST + Makefile.PL META.json META.yml - Makefile.PL + minil.toml NEWS README README.markdown @@ -106,12 +118,47 @@ sub interesting_files { README.mkdn THANKS TODO + TODO.md ToDo + ToDo.md Todo - cpanfile - alienfile - dist.ini - minil.toml + Todo.md + ) + ] + } + } + ] + } + }, + { + bool => { + must => [ + { + terms => { + name => [ + qw( + CONTRIBUTING.pm + CONTRIBUTING.pod + Contributing.pm + Contributing.pod + ChangeLog.pm + ChangeLog.pod + Changelog.pm + Changelog.pod + CHANGES.pm + CHANGES.pod + Changes.pm + Changes.pod + HACKING.pm + HACKING.pod + Hacking.pm + Hacking.pod + TODO.pm + TODO.pod + ToDo.pm + ToDo.pod + Todo.pm + Todo.pod ) ] } From 0fa860db70703ddcd012c47f5891021636671990 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 24 May 2018 13:58:41 +0200 Subject: [PATCH 006/892] update changes picking code to match interesting files Update the list of files to pick for a changelog to match the relevant set of files listed in interesting files. Also allow .pod and .pm files in subdirectories to be picked, but prioritize files in the root directory. --- lib/MetaCPAN/Model/Release.pm | 49 +++++++++++++++++++++++++++-------- 1 file changed, 38 insertions(+), 11 deletions(-) diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index b2f2f5bf0..be5ff2b09 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -267,18 +267,41 @@ sub set_main_module { } -sub get_changes_file { - my $self = shift; - my @files = @{ $_[0] }; - my @changes_files = qw( - Changelog - ChangeLog - CHANGELOG - Changes - CHANGES - NEWS +my @changes_files = qw( + CHANGELOG + ChangeLog + Changelog + CHANGES + Changes + NEWS +); +my @exclude_dirs = qw( + corpus + fatlib + inc + local + perl5 + share + t + xt +); + +# this should match the same set of files as MetaCPAN::Query::File->interesting_files +my ($changes_match) = map qr/^(?:$_)$/, join '|', + ( map quotemeta, @changes_files ), + ( + "(?:(?!" + . join( '|', map "$_/", @exclude_dirs ) + . ").*/)?(?:" + . join( + '|', map quotemeta, map +( "$_.pm", "$_.pod" ), @changes_files + ) + . ')' ); +sub get_changes_file { + my $self = shift; + my @files = @{ $_[0] }; if ( $files[0]->distribution eq 'perl' ) { foreach my $file (@files) { if ( $file->name eq 'perldelta.pod' ) { @@ -286,8 +309,12 @@ sub get_changes_file { } } } + + # prioritize files in the top level but otherwise alphabetical + @files = sort { $a->level <=> $b->level || $a->path cmp $b->path } @files; + foreach my $file (@files) { - return $file->path if grep { $_ eq $file->path } @changes_files; + return $file->path if $file->path =~ $changes_match; } } From e40c67e3da32b0894745396fb50c4f2b469d176a Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Wed, 23 May 2018 12:46:05 +0200 Subject: [PATCH 007/892] sort cpanfile --- cpanfile | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/cpanfile b/cpanfile index b0b67d735..004e3dc51 100644 --- a/cpanfile +++ b/cpanfile @@ -8,6 +8,7 @@ requires 'CPAN::DistnameInfo', '0.12'; requires 'CPAN::Meta', '2.150005'; # Avoid issues with List::Util dep under carton install. requires 'CPAN::Meta::Requirements', '2.140'; requires 'CPAN::Meta::YAML', '0.018'; +requires 'CPAN::Repository::Perms'; requires 'Captcha::reCAPTCHA', '0.99'; requires 'Catalyst', '5.90103'; requires 'Catalyst::Action::RenderView'; @@ -25,20 +26,19 @@ requires 'Catalyst::Utils'; requires 'Catalyst::View'; requires 'Catalyst::View::JSON', '0.36'; requires 'CatalystX::Component::Traits'; +requires 'CatalystX::Fastly::Role::Response', '0.06'; requires 'CatalystX::InjectComponent'; requires 'CatalystX::RoleApplicator'; -requires 'CatalystX::Fastly::Role::Response', '0.06'; -requires 'CPAN::Repository::Perms'; requires 'Config::ZOMG', '>=', '1.000000'; requires 'Const::Fast'; requires 'Cpanel::JSON::XS', '3.0115'; requires 'Cwd'; -requires 'Data::Printer', '0.38'; requires 'DBD::SQLite', '>=1.50'; requires 'DBI', '1.616'; requires 'Data::DPath'; requires 'Data::Dump'; requires 'Data::Dumper'; +requires 'Data::Printer', '0.38'; requires 'DateTime', '1.24'; requires 'DateTime::Format::ISO8601'; requires 'Devel::ArgNames'; @@ -99,18 +99,18 @@ requires 'Module::Load'; requires 'Module::Metadata', '1.000022'; requires 'Module::Pluggable'; requires 'Module::Runtime'; -requires 'Moose', ' >= 2.1403'; requires 'Mojo::Pg', '>= 4.08'; +requires 'Moose', ' >= 2.1403'; requires 'Moose::Role'; requires 'Moose::Util'; requires 'MooseX::Aliases'; requires 'MooseX::Attribute::Deflator', '2.1.5'; requires 'MooseX::ChainedAccessors'; requires 'MooseX::ClassAttribute'; +requires 'MooseX::Fastly::Role', '0.02'; requires 'MooseX::Getopt', '0.71'; requires 'MooseX::Getopt::Dashes'; requires 'MooseX::Getopt::OptionTypeMap'; -requires 'MooseX::Fastly::Role', '0.02'; requires 'MooseX::StrictConstructor'; requires 'MooseX::Types'; requires 'MooseX::Types::Common::String'; @@ -130,8 +130,8 @@ requires 'Parse::CPAN::Packages::Fast', '0.09'; requires 'Parse::CSV', '2.04'; requires 'Parse::PMFile', '0.41'; requires 'Path::Class', '>= 0.36'; -requires 'Path::Iterator::Rule', '>=1.011'; requires 'Path::Class::File'; +requires 'Path::Iterator::Rule', '>=1.011'; requires 'PerlIO::gzip'; requires 'Pithub', '0.01033'; requires 'Plack', '1.0039'; @@ -158,8 +158,8 @@ requires 'Regexp::Common::time'; requires 'Safe', '2.35'; # bug fixes (used by Parse::PMFile) requires 'Search::Elasticsearch', '== 2.03'; requires 'Starman'; -requires 'Time::Local'; requires 'Throwable::Error'; +requires 'Time::Local'; requires 'Try::Tiny', '0.24'; requires 'URI', '1.71'; requires 'URI::Escape'; @@ -178,17 +178,17 @@ requires 'version', '0.9901'; requires 'warnings'; test_requires 'App::Prove'; +test_requires 'CPAN::Faker', '0.010'; test_requires 'Code::TidyAll', '>= 0.47'; test_requires 'Code::TidyAll::Plugin::UniqueLines'; -test_requires 'CPAN::Faker', '0.010'; -test_requires 'Devel::Confess'; -test_requires 'Module::Faker', '0.015'; -test_requires 'Module::Faker::Dist', '0.010'; test_requires 'Config::General'; +test_requires 'Devel::Confess'; test_requires 'File::Copy'; test_requires 'HTTP::Cookies'; test_requires 'LWP::ConsoleLogger::Easy'; test_requires 'MetaCPAN::Client', '>=', '2.017000'; +test_requires 'Module::Faker', '0.015'; +test_requires 'Module::Faker::Dist', '0.010'; test_requires 'Perl::Tidy' => '20180220'; test_requires 'Plack::Test::Agent'; test_requires 'Test::Code::TidyAll'; From aa133c5f1fcc8ba91cf65ce318ba36c794f11ff4 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 28 May 2018 18:05:16 +0200 Subject: [PATCH 008/892] author prefix search --- lib/MetaCPAN/Document/Author/Set.pm | 2 +- lib/MetaCPAN/Query/Author.pm | 33 ++++++++++++++++++++++++ lib/MetaCPAN/Server/Controller/Author.pm | 10 +++++++ 3 files changed, 44 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Document/Author/Set.pm b/lib/MetaCPAN/Document/Author/Set.pm index 39571eb22..2120ecd2b 100644 --- a/lib/MetaCPAN/Document/Author/Set.pm +++ b/lib/MetaCPAN/Document/Author/Set.pm @@ -11,7 +11,7 @@ has query_author => ( isa => 'MetaCPAN::Query::Author', lazy => 1, builder => '_build_query_author', - handles => [qw< by_ids by_user search >], + handles => [qw< by_ids by_user search prefix_search >], ); sub _build_query_author { diff --git a/lib/MetaCPAN/Query/Author.pm b/lib/MetaCPAN/Query/Author.pm index 6d7c957ff..52ebe3472 100644 --- a/lib/MetaCPAN/Query/Author.pm +++ b/lib/MetaCPAN/Query/Author.pm @@ -105,5 +105,38 @@ sub search { }; } +sub prefix_search { + my ( $self, $query, $opts ) = @_; + my $size = $opts->{size} // 500; + my $from = $opts->{from} // 0; + + my $body = { + query => { + prefix => { + pauseid => $query, + }, + }, + size => $size, + from => $from, + }; + + my $ret = $self->es->search( + index => $self->index_name, + type => 'author', + body => $body, + ); + + my @authors = map { + single_valued_arrayref_to_scalar( $_->{_source} ); + +{ %{ $_->{_source} }, id => $_->{_id} } + } @{ $ret->{hits}{hits} }; + + return +{ + authors => \@authors, + took => $ret->{took}, + total => $ret->{hits}{total}, + }; +} + __PACKAGE__->meta->make_immutable; 1; diff --git a/lib/MetaCPAN/Server/Controller/Author.pm b/lib/MetaCPAN/Server/Controller/Author.pm index b309ae55e..cc5ad1e76 100644 --- a/lib/MetaCPAN/Server/Controller/Author.pm +++ b/lib/MetaCPAN/Server/Controller/Author.pm @@ -61,4 +61,14 @@ sub by_users : Path('by_user') : Args(0) { $self->model($c)->by_user( $c->read_param('user') ) ); } +# /author/by_prefix/PAUSE_ID_PREFIX +sub by_prefix : Path('by_prefix') : Args(1) { + my ( $self, $c, $prefix ) = @_; + my ($size) = $c->read_param('size')->[0] // 500; + my ($from) = $c->read_param('from')->[0] // 0; + + $c->stash_or_detach( $self->model($c) + ->prefix_search( $prefix, { size => $size, from => $from } ) ); +} + 1; From 9e9aac8525706d6b25baf971ec3b0144d2048a94 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 28 May 2018 23:09:21 +0200 Subject: [PATCH 009/892] support .md files for change log files --- lib/MetaCPAN/Model/Release.pm | 2 +- lib/MetaCPAN/Query/File.pm | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index be5ff2b09..5a03d645f 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -288,7 +288,7 @@ my @exclude_dirs = qw( # this should match the same set of files as MetaCPAN::Query::File->interesting_files my ($changes_match) = map qr/^(?:$_)$/, join '|', - ( map quotemeta, @changes_files ), + ( map quotemeta, @changes_files, map "$_.md", @changes_files ), ( "(?:(?!" . join( '|', map "$_/", @exclude_dirs ) diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index 3fa86cba5..62b4039ee 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -78,10 +78,15 @@ sub interesting_files { AUTHORS Build.PL CHANGELOG + CHANGELOG.md ChangeLog + ChangeLog.md Changelog + Changelog.md CHANGES + CHANGES.md Changes + Changes.md CONTRIBUTING CONTRIBUTING.md Contributing @@ -111,6 +116,7 @@ sub interesting_files { META.yml minil.toml NEWS + NEWS.md README README.markdown README.md From 700541c232d423d7d4589a50900b48e2dd9d3a59 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 28 May 2018 23:28:31 +0200 Subject: [PATCH 010/892] show build log on prereq failure --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f298bade1..f3672f000 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,7 @@ before_install: - cpanm -n Safe@2.35 install: - - cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) + - cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (cat ~/.perl-cpm/build.log; false) before_script: - "perl -i -pe 's/(servers :)9900/localhost:9200/' metacpan_server_testing.conf" From c63e1a25bcea5dbf803fff43716e095241ae1c1d Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 28 May 2018 23:41:18 +0200 Subject: [PATCH 011/892] no author testing for prereqs --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index f3672f000..3fa01ded4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,7 @@ before_install: - cpanm -n Safe@2.35 install: - - cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (cat ~/.perl-cpm/build.log; false) + - AUTHOR_TESTING=0 cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (cat ~/.perl-cpm/build.log; false) before_script: - "perl -i -pe 's/(servers :)9900/localhost:9200/' metacpan_server_testing.conf" From ebdbc0d674c3623ec23c98cb995b110c1c681066 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 3 Jun 2018 21:17:46 +0200 Subject: [PATCH 012/892] allow requesting more user details --- lib/MetaCPAN/Query/Author.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Query/Author.pm b/lib/MetaCPAN/Query/Author.pm index 52ebe3472..609cd8d8a 100644 --- a/lib/MetaCPAN/Query/Author.pm +++ b/lib/MetaCPAN/Query/Author.pm @@ -45,7 +45,7 @@ sub by_user { type => 'author', body => { query => { terms => { user => $users } }, - size => 100, + size => 500, } ); return {} unless $authors->{hits}{total}; From 7e3cdeec54767a90bbe40ee2b6f161d42d40cf21 Mon Sep 17 00:00:00 2001 From: Joelle Maslak Date: Mon, 11 Jun 2018 11:47:27 -0600 Subject: [PATCH 013/892] Minor changes to make the API docs more accessible Some developers that may use this API may not be familiar with terms like API and DSL, or even why you would care about an API. --- docs/API-docs.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/API-docs.md b/docs/API-docs.md index d826deb7a..c74a7c203 100644 --- a/docs/API-docs.md +++ b/docs/API-docs.md @@ -1,12 +1,12 @@ # API Docs: v1 -For an introduction to the MetaCPAN API which requires no previous knowledge of MetaCPAN or ElasticSearch, see [the slides for "Abusing MetaCPAN for Fun and Profit"](https://www.slideshare.net/oalders/abusing-metacpan2013) or [watch the actual talk](https://www.youtube.com/watch?v=J8ymBuFlHQg). +For an introduction to the MetaCPAN API (Application Program Interface) which requires no previous knowledge of MetaCPAN or ElasticSearch, see [the slides for "Abusing MetaCPAN for Fun and Profit"](https://www.slideshare.net/oalders/abusing-metacpan2013) or [watch the actual talk](https://www.youtube.com/watch?v=J8ymBuFlHQg). This API lets you programmatically access MetaCPAN from your own applications. There is also [a repository of examples](https://github.com/metacpan/metacpan-examples) you can play with to get up and running in a hurry. Rather than editing this wiki page, please send pull requests for the metacpan-examples repository. If you'd rather edit the wiki, please do, but sending the code pull requests is probably the most helpful way to approach this. _All of these URLs can be tested using the [MetaCPAN Explorer](https://explorer.metacpan.org)_ -To learn more about the ElasticSearch query DSL check out Clinton Gormley's [Terms of Endearment - ES Query DSL Explained] (https://www.slideshare.net/clintongormley/terms-of-endearment-the-elasticsearch-query-dsl-explained) slides. +To learn more about the ElasticSearch query DSL (Domain-Specific Language) check out Clinton Gormley's [Terms of Endearment - ES Query DSL Explained] (https://www.slideshare.net/clintongormley/terms-of-endearment-the-elasticsearch-query-dsl-explained) slides. The query syntax is explained on ElasticSearch's [reference page](https://www.elasticsearch.org/guide/reference/query-dsl/). You can also check out this getting started tutorial about Elasticsearch [reference page](https://joelabrahamsson.com/elasticsearch-101/). From 416f26f757ea35f63a164735a42eeacabefbcdf9 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 15 Jun 2018 12:11:38 -0500 Subject: [PATCH 014/892] minor cleanups to search code --- lib/MetaCPAN/Model/Search.pm | 30 +++++++++++++++--------------- 1 file changed, 15 insertions(+), 15 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 6b767c4dc..a494b696f 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -109,7 +109,6 @@ sub _search_expanded { } ); - #return $es_query; my $es_results = $self->run_query( file => $es_query ); my @distributions = uniq @@ -156,9 +155,8 @@ sub _search_collapsed { # We need to scan enough modules to build up a sufficient number of # distributions to fill the results to the number requested my $es_query_opts = { - size => $RESULTS_PER_RUN, - from => ( $run - 1 ) * $RESULTS_PER_RUN, - fields => [qw(distribution)], + size => $RESULTS_PER_RUN, + from => ( $run - 1 ) * $RESULTS_PER_RUN, }; if ( $run == 1 ) { @@ -401,21 +399,23 @@ sub build_query { } } }, - _source => "module", - fields => [ + _source => [ + "module", + ], + fields => [ qw( - documentation - author abstract.analyzed - release - path - status - indexed + author authorized - distribution date + distribution + documentation id + indexed + path pod_lines + release + status ) ], } @@ -531,10 +531,10 @@ sub _extract_results_add_favs { my $res = $_; single_valued_arrayref_to_scalar( $res->{fields} ); +{ + abstract => delete $res->{fields}->{'abstract.analyzed'}, %{ $res->{fields} }, %{ $res->{_source} }, - abstract => delete $res->{fields}->{'abstract.analyzed'}, - score => $res->{_score}, + score => $res->{_score}, favorites => $favorites->{favorites}{ $res->{fields}->{distribution} }, } From 2ad73a657674374dd65452c1cb61c41f7180a00c Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 15 Jun 2018 12:15:45 -0500 Subject: [PATCH 015/892] fetch favorite counts directly in search query The file index now contains the dist favorite counts, so we can fetch them from there rather than doing a secondary lookup. --- lib/MetaCPAN/Model/Search.pm | 86 ++++++------------------------------ 1 file changed, 13 insertions(+), 73 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index a494b696f..7dac14f5a 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -111,18 +111,11 @@ sub _search_expanded { my $es_results = $self->run_query( file => $es_query ); - my @distributions = uniq - map { - single_valued_arrayref_to_scalar( $_->{fields} ); - $_->{fields}->{distribution} - } @{ $es_results->{hits}->{hits} }; - # Everything after this will fail (slowly and silently) without results. - return {} unless @distributions; + return {} unless @{ $es_results->{hits}{hits} }; - # Lookup favs and extract results from es (adding in favs) - my $favorites = $self->search_favorites(@distributions); - my $results = $self->_extract_results_add_favs( $es_results, $favorites ); + # Extract results from es + my $results = $self->_extract_results($es_results); # Add descriptions my @ids = map { $_->{id} } @{$results}; @@ -132,9 +125,9 @@ sub _search_expanded { @{$results}; my $return = { - results => [ map { [$_] } @$results ], - total => $es_results->{hits}->{total}, - took => sum( grep {defined} $es_results->{took}, $favorites->{took} ), + results => [ map { [$_] } @$results ], + total => $es_results->{hits}->{total}, + took => $es_results->{took}, collapsed => \0, }; return $return; @@ -233,10 +226,7 @@ sub _search_collapsed { ); my $es_dist_results = $self->run_query( file => $es_query ); - # Look up favs and add to extracted results - my $favorites = $self->search_favorites(@distributions); - my $results - = $self->_extract_results_add_favs( $es_dist_results, $favorites ); + my $results = $self->_extract_results($es_dist_results); $results = $self->_collapse_results($results); # Add descriptions, but only after collapsed as is slow @@ -247,7 +237,7 @@ sub _search_collapsed { # Calculate took from sum of all ES searches $took += sum( grep {defined} $es_dist_results->{took}, - $favorites->{took}, $descriptions->{took} ); + $descriptions->{took} ); return { results => $results, @@ -408,6 +398,7 @@ sub build_query { author authorized date + dist_fav_count distribution documentation id @@ -473,70 +464,19 @@ sub search_descriptions { return $results; } -sub _build_search_favorites_query { - my ( $self, @distributions ) = @_; - - my $es_query = { - size => 0, - query => { - filtered => { - query => { match_all => {} }, - filter => { - or => [ - map { { term => { 'distribution' => $_ } } } - @distributions - ] - } - } - }, - aggregations => { - favorites => { - terms => { - field => 'distribution', - size => scalar @distributions, - }, - }, - } - }; - - return $es_query; -} - -sub search_favorites { - my ( $self, @distributions ) = @_; - @distributions = uniq @distributions; - - # If there are no distributions this will build a query with an empty - # filter and ES will return a parser error... so just skip it. - return {} unless @distributions; - - my $es_query = $self->_build_search_favorites_query(@distributions); - my $es_results = $self->run_query( favorite => $es_query ); - - my $results = { - took => $es_results->{took}, - favorites => { - map { $_->{key} => $_->{doc_count} } - @{ $es_results->{aggregations}->{favorites}->{buckets} } - }, - }; - return $results; -} - -sub _extract_results_add_favs { - my ( $self, $es_results, $favorites ) = @_; +sub _extract_results { + my ( $self, $es_results ) = @_; return [ map { my $res = $_; single_valued_arrayref_to_scalar( $res->{fields} ); +{ - abstract => delete $res->{fields}->{'abstract.analyzed'}, + abstract => delete $res->{fields}->{'abstract.analyzed'}, + favorites => delete $res->{fields}->{dist_fav_count}, %{ $res->{fields} }, %{ $res->{_source} }, score => $res->{_score}, - favorites => - $favorites->{favorites}{ $res->{fields}->{distribution} }, } } @{ $es_results->{hits}{hits} } ]; From 505125f7c86609b451020ed96d59500ce0bc9bf1 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 15 Jun 2018 12:20:09 -0500 Subject: [PATCH 016/892] fetch descriptions directly in search Avoid using a secondary query. --- lib/MetaCPAN/Model/Search.pm | 56 ++---------------------------------- 1 file changed, 2 insertions(+), 54 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 7dac14f5a..c916d0901 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -111,19 +111,9 @@ sub _search_expanded { my $es_results = $self->run_query( file => $es_query ); - # Everything after this will fail (slowly and silently) without results. - return {} unless @{ $es_results->{hits}{hits} }; - # Extract results from es my $results = $self->_extract_results($es_results); - # Add descriptions - my @ids = map { $_->{id} } @{$results}; - my $descriptions = $self->search_descriptions(@ids); - - map { $_->{description} = $descriptions->{results}->{ $_->{id} } } - @{$results}; - my $return = { results => [ map { [$_] } @$results ], total => $es_results->{hits}->{total}, @@ -229,15 +219,7 @@ sub _search_collapsed { my $results = $self->_extract_results($es_dist_results); $results = $self->_collapse_results($results); - # Add descriptions, but only after collapsed as is slow - my @ids = map { $_->[0]{id} } @$results; - my $descriptions = $self->search_descriptions(@ids); - map { $_->[0]{description} = $descriptions->{results}{ $_->[0]{id} } } - @{$results}; - - # Calculate took from sum of all ES searches - $took += sum( grep {defined} $es_dist_results->{took}, - $descriptions->{took} ); + $took += $es_dist_results->{took}; return { results => $results, @@ -398,6 +380,7 @@ sub build_query { author authorized date + description dist_fav_count distribution documentation @@ -429,41 +412,6 @@ sub run_query { ); } -sub _build_search_descriptions_query { - my ( $self, @ids ) = @_; - my $es_query = { - query => { - filtered => { - query => { match_all => {} }, - filter => { or => [ map { { term => { id => $_ } } } @ids ] } - } - }, - fields => [qw(description id)], - size => scalar @ids, - }; - return $es_query; -} - -sub search_descriptions { - my ( $self, @ids ) = @_; - return { - descriptions => {}, - took => 0, - } unless @ids; - - my $es_query = $self->_build_search_descriptions_query(@ids); - my $es_results = $self->run_query( file => $es_query ); - my $results = { - results => { - map { $_->{id} => $_->{description} } - map { single_valued_arrayref_to_scalar( $_->{fields} ) } - @{ $es_results->{hits}->{hits} } - }, - took => $es_results->{took} - }; - return $results; -} - sub _extract_results { my ( $self, $es_results ) = @_; From 02ebd012c52a96ec5c0795c4d30fdfb8a8cd7668 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 15 Jun 2018 12:21:28 -0500 Subject: [PATCH 017/892] perform collapsed search with ES aggregation Elasticsearch aggregations can do the same type of grouping we have been doing with multiple fetches and collapsing the results. This allows the entire search to be done in a single ES query. --- lib/MetaCPAN/Model/Search.pm | 172 ++++++++++++----------------------- 1 file changed, 57 insertions(+), 115 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index c916d0901..b417ef78c 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -126,128 +126,70 @@ sub _search_expanded { sub _search_collapsed { my ( $self, $search_term, $from, $page_size ) = @_; - my $total; - my @distributions; - my $es_results; - - my $run = 1; - my $hits = 0; - my $took = 0; - - do { - # We need to scan enough modules to build up a sufficient number of - # distributions to fill the results to the number requested - my $es_query_opts = { - size => $RESULTS_PER_RUN, - from => ( $run - 1 ) * $RESULTS_PER_RUN, - }; - - if ( $run == 1 ) { - - # On the first request also fetch the number of total distributions - # that match the query so that can be reported to the user. There is - # no need to do it on each iteration though, once is enough. - - $es_query_opts->{aggregations} - = { - count => { terms => { size => 999, field => 'distribution' } } - }; - } - - my $es_query = $self->build_query( $search_term, $es_query_opts ); - $es_results = $self->run_query( file => $es_query ); - $took += $es_results->{took} || 0; - - if ( $run == 1 ) { - $total - = @{ $es_results->{aggregations}->{count}->{buckets} || [] }; - } - - $hits = @{ $es_results->{hits}->{hits} || [] }; - - # Flatten results down to unique dists - @distributions = uniq( - @distributions, - map { - single_valued_arrayref_to_scalar( $_->{fields} ); - $_->{fields}->{distribution} - } @{ $es_results->{hits}->{hits} } - ); - - # Keep track - $run++; - } while ( @distributions < $page_size + $from - && $es_results->{hits}->{total} - && $es_results->{hits}->{total} - > $hits + ( $run - 2 ) * $RESULTS_PER_RUN ); - - # Avoid "splice() offset past end of array" warning. - @distributions - = $from > @distributions - ? () - : splice( @distributions, $from, $page_size ); - - # Everything else will fail (slowly and quietly) without distributions. - return {} unless @distributions; - - # Now that we know which distributions are going to be displayed on the - # results page, fetch the details about those distributions - my $es_query = $self->build_query( - $search_term, - { -# we will probably never hit that limit, since we are searching in $page_size=20 distributions max - size => 5000, - query => { - filtered => { - filter => { - and => [ - { - or => [ - map { - { term => { 'distribution' => $_ } } - } @distributions - ] - } - ] - } - } - } - } - ); - my $es_dist_results = $self->run_query( file => $es_query ); + my $total_size = $from + $page_size; + + my $es_query_opts = { + size => 0, + fields => [ + qw( + ) + ], + }; - my $results = $self->_extract_results($es_dist_results); - $results = $self->_collapse_results($results); + my $es_query = $self->build_query( $search_term, $es_query_opts ); + my $fields = delete $es_query->{fields}; + my $source = delete $es_query->{_source}; + + $es_query->{aggregations} = { + by_dist => { + terms => { + size => $total_size, + field => 'distribution', + order => { + max_score => 'desc', + }, + }, + aggregations => { + top_modules => { + top_hits => { + fields => $fields, + _source => $source, + size => 500, + }, + }, + max_score => { + max => { + lang => "expression", + script => "_score", + }, + }, + }, + }, + total_dists => { + cardinality => { + field => 'distribution', + }, + }, + }; - $took += $es_dist_results->{took}; + my $es_results = $self->run_query( file => $es_query ); - return { - results => $results, - total => $total, - took => $took, + my $output = { + results => [], + total => $es_results->{aggregations}{total_dists}{value}, + took => $es_results->{took}, collapsed => \1, }; -} + my @dists = @{ $es_results->{aggregations}{by_dist}{buckets} } + [ $from .. $total_size - 1 ]; -sub _collapse_results { - my ( $self, $results ) = @_; - my %collapsed; - foreach my $result (@$results) { - my $distribution = $result->{distribution}; - $collapsed{$distribution} - = { position => scalar keys %collapsed, results => [] } - unless ( $collapsed{$distribution} ); - push( @{ $collapsed{$distribution}->{results} }, $result ); - } - - # We return array ref because the results have matching modules - # grouped by distribution - return [ - map { $collapsed{$_}->{results} } - sort { $collapsed{$a}->{position} <=> $collapsed{$b}->{position} } - keys %collapsed - ]; + @{ $output->{results} } = map { + my $dist = $_; + $self->_extract_results( $_->{top_modules} ); + } @dists; + + return $output; } sub build_query { From ba9ad6e8c334f5d9fc2f73e87a7f5678226084c6 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 15 Jun 2018 16:02:52 -0500 Subject: [PATCH 018/892] fix handling for incomplete pages of search results --- lib/MetaCPAN/Model/Search.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index b417ef78c..7c6e274db 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -7,7 +7,7 @@ use Log::Contextual qw( :log :dlog ); use MooseX::StrictConstructor; use Hash::Merge qw( merge ); -use List::Util qw( sum uniq ); +use List::Util qw( min uniq ); use MetaCPAN::Types qw( Object Str ); use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); @@ -181,8 +181,10 @@ sub _search_collapsed { collapsed => \1, }; + my $last = min( $total_size - 1, + $#{ $es_results->{aggregations}{by_dist}{buckets} } ); my @dists = @{ $es_results->{aggregations}{by_dist}{buckets} } - [ $from .. $total_size - 1 ]; + [ $from .. $last ]; @{ $output->{results} } = map { my $dist = $_; From 171d9ce5059d271df5139895bfe6bceb37a4154d Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 16 Jun 2018 08:33:15 -0500 Subject: [PATCH 019/892] fix search test --- lib/MetaCPAN/Model/Search.pm | 5 ++-- t/model/search.t | 50 ++++++++++-------------------------- 2 files changed, 17 insertions(+), 38 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 7c6e274db..0ed98082e 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -5,6 +5,7 @@ use MetaCPAN::Moose; use Const::Fast qw( const ); use Log::Contextual qw( :log :dlog ); use MooseX::StrictConstructor; +use Cpanel::JSON::XS (); use Hash::Merge qw( merge ); use List::Util qw( min uniq ); @@ -118,7 +119,7 @@ sub _search_expanded { results => [ map { [$_] } @$results ], total => $es_results->{hits}->{total}, took => $es_results->{took}, - collapsed => \0, + collapsed => Cpanel::JSON::XS::false(), }; return $return; } @@ -178,7 +179,7 @@ sub _search_collapsed { results => [], total => $es_results->{aggregations}{total_dists}{value}, took => $es_results->{took}, - collapsed => \1, + collapsed => Cpanel::JSON::XS::true(), }; my $last = min( $total_size - 1, diff --git a/t/model/search.t b/t/model/search.t index b17d3b95b..cc4fe2ade 100644 --- a/t/model/search.t +++ b/t/model/search.t @@ -6,6 +6,7 @@ use MetaCPAN::Model::Search (); use MetaCPAN::TestServer (); use Test::More; use Test::Deep qw(cmp_deeply ignore); +use Cpanel::JSON::XS (); plan skip_all => "Travis ES bad, see https://travis-ci.org/metacpan/metacpan-api/jobs/301092129" @@ -23,7 +24,16 @@ ok( $search->_not_rogue, '_not_rogue' ); { my $results = $search->search_web('Fooxxxx'); - cmp_deeply( $results, {}, 'no results on fake module' ); + cmp_deeply( + $results, + { + results => [], + total => 0, + took => ignore(), + collapsed => Cpanel::JSON::XS::true(), + }, + 'no results on fake module' + ); } { @@ -31,10 +41,7 @@ ok( $search->_not_rogue, '_not_rogue' ); is( scalar @{ $collapsed_search->{results}->[0] }, 2, 'got results for collapsed search' ); - ok( - ${ $collapsed_search->{collapsed} }, - 'results are flagged as collapsed' - ); + ok( $collapsed_search->{collapsed}, 'results are flagged as collapsed' ); my $from = 0; my $page_size = 20; @@ -43,7 +50,7 @@ ok( $search->_not_rogue, '_not_rogue' ); my $expanded = $search->search_web( 'Foo', $from, $page_size, $collapsed ); - ok( !${ $expanded->{collapsed} }, 'results are flagged as expanded' ); + ok( !$expanded->{collapsed}, 'results are flagged as expanded' ); is( $expanded->{results}->[0]->[0]->{path}, 'lib/Pod/Pm.pm', 'first expanded result is expected' ); @@ -71,39 +78,10 @@ ok( $search->_not_rogue, '_not_rogue' ); my $module = 'Binary::Data::WithPod'; my $results = $search->search_web($module); is( - $results->{results}->[0]->[0]->{description}, + $results->{results}->[0]->{hits}->[0]->{description}, 'razzberry pudding', 'description included in results' ); } -{ - my $id = 'JatCtNR2RGjcBIs1Y5C_zTzNcXU'; - my $results = $search->search_descriptions($id); - cmp_deeply( $results->{results}, { $id => 'TBD' }, - 'search_descriptions' ); -} - -# favorites are also tested in t/server/controller/user/favorite.t -cmp_deeply( $search->search_favorites, {}, - 'empty hashref when no distributions' ); - -cmp_deeply( - $search->search_favorites('Pod-Pm'), - { - favorites => {}, - took => ignore(), - }, - 'no favorites found' -); - -cmp_deeply( - $search->search_descriptions, - { - descriptions => {}, - took => ignore(), - }, - 'empty hashref when no ids for descriptions' -); - done_testing(); From c3dd294be1ebccdea08e132c1ce9dca87c52a902 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 15 Jun 2018 16:25:24 -0500 Subject: [PATCH 020/892] new search end point with total collapsed entries The existing search end points gives an array of arrays. For collapsed searches, each inner array represents a dist. Because these arrays are stored directly without a wrapping object, there is nowhere to store additional data about the results. For collapsed results, there is little reason to include all of the inner file results. For the metacpan front end, we only need the first 5 or so, as well as the count. There isn't anywhere to store the count without introducing a wrapping object for each search result. The new end point returns an array of objects. The objects include the distribution name, the top five inner hits, and the total amount of inner hits. --- lib/MetaCPAN/Model/Search.pm | 34 +++++++++++++++----- lib/MetaCPAN/Server/Controller/Search/Web.pm | 15 +++++++++ t/model/search.t | 6 ++-- 3 files changed, 44 insertions(+), 11 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 0ed98082e..bc357e31e 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -72,7 +72,9 @@ sub search_for_first_result { =cut sub search_web { - my ( $self, $search_term, $from, $page_size, $collapsed ) = @_; + my ( $self, $search_term, $from, $page_size, $collapsed, + $max_collapsed_hits ) + = @_; $page_size //= 20; $from //= 0; @@ -90,7 +92,8 @@ sub search_web { my $results = $collapsed // $search_term !~ /(distribution|module\.name\S*):/ - ? $self->_search_collapsed( $search_term, $from, $page_size ) + ? $self->_search_collapsed( $search_term, $from, $page_size, + $max_collapsed_hits ) : $self->_search_expanded( $search_term, $from, $page_size ); return $results; @@ -115,8 +118,18 @@ sub _search_expanded { # Extract results from es my $results = $self->_extract_results($es_results); + $results = [ + map { + { + hits => [$_], + distribution => $_->{distribution}, + total => 1, + } + } @$results + ]; + my $return = { - results => [ map { [$_] } @$results ], + results => $results, total => $es_results->{hits}->{total}, took => $es_results->{took}, collapsed => Cpanel::JSON::XS::false(), @@ -125,7 +138,9 @@ sub _search_expanded { } sub _search_collapsed { - my ( $self, $search_term, $from, $page_size ) = @_; + my ( $self, $search_term, $from, $page_size, $max_collapsed_hits ) = @_; + + $max_collapsed_hits ||= 5; my $total_size = $from + $page_size; @@ -151,11 +166,11 @@ sub _search_collapsed { }, }, aggregations => { - top_modules => { + top_files => { top_hits => { fields => $fields, _source => $source, - size => 500, + size => $max_collapsed_hits, }, }, max_score => { @@ -188,8 +203,11 @@ sub _search_collapsed { [ $from .. $last ]; @{ $output->{results} } = map { - my $dist = $_; - $self->_extract_results( $_->{top_modules} ); + +{ + hits => $self->_extract_results( $_->{top_files} ), + distribution => $_->{key}, + total => $_->{doc_count}, + }; } @dists; return $output; diff --git a/lib/MetaCPAN/Server/Controller/Search/Web.pm b/lib/MetaCPAN/Server/Controller/Search/Web.pm index a4781e903..8c85615a9 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Web.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Web.pm @@ -43,6 +43,21 @@ sub web : Chained('/search/index') : PathPart('web') : Args(0) { my ( $self, $c ) = @_; my $args = $c->req->params; + my $model = $c->model('Search'); + my $results + = $model->search_web( @{$args}{qw( q from size collapsed )}, 500 ); + + for my $result ( @{ $results->{results} } ) { + $result = $result->{hits}; + } + + $c->stash($results); +} + +sub web_v2 : Chained('/search/index') : PathPart('web/v2') : Args(0) { + my ( $self, $c ) = @_; + my $args = $c->req->params; + my $model = $c->model('Search'); my $results = $model->search_web( @{$args}{qw( q from size collapsed )} ); diff --git a/t/model/search.t b/t/model/search.t index cc4fe2ade..87c5ad516 100644 --- a/t/model/search.t +++ b/t/model/search.t @@ -38,7 +38,7 @@ ok( $search->_not_rogue, '_not_rogue' ); { my $collapsed_search = $search->search_web('Foo'); - is( scalar @{ $collapsed_search->{results}->[0] }, + is( scalar @{ $collapsed_search->{results}->[0]->{hits} }, 2, 'got results for collapsed search' ); ok( $collapsed_search->{collapsed}, 'results are flagged as collapsed' ); @@ -52,9 +52,9 @@ ok( $search->_not_rogue, '_not_rogue' ); ok( !$expanded->{collapsed}, 'results are flagged as expanded' ); - is( $expanded->{results}->[0]->[0]->{path}, + is( $expanded->{results}->[0]->{hits}->[0]->{path}, 'lib/Pod/Pm.pm', 'first expanded result is expected' ); - is( $expanded->{results}->[1]->[0]->{path}, + is( $expanded->{results}->[1]->{hits}->[0]->{path}, 'lib/Pod/Pm/NoPod.pod', 'second expanded result is expected' ); } From ed925b1a833ed6d644fb2459cb1c186da2998baa Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 16 Jun 2018 13:08:05 -0500 Subject: [PATCH 021/892] fix search model test --- t/model/search.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/model/search.t b/t/model/search.t index cc4fe2ade..20d2b50f9 100644 --- a/t/model/search.t +++ b/t/model/search.t @@ -78,7 +78,7 @@ ok( $search->_not_rogue, '_not_rogue' ); my $module = 'Binary::Data::WithPod'; my $results = $search->search_web($module); is( - $results->{results}->[0]->{hits}->[0]->{description}, + $results->{results}->[0]->[0]->{description}, 'razzberry pudding', 'description included in results' ); From 2d6f13ac7b1c0b98c15757e7afe23e3c1516179d Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 16 Jun 2018 22:09:06 -0500 Subject: [PATCH 022/892] size and from should be optional in author/by_prefix --- lib/MetaCPAN/Server/Controller/Author.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Author.pm b/lib/MetaCPAN/Server/Controller/Author.pm index cc5ad1e76..6f42b4a58 100644 --- a/lib/MetaCPAN/Server/Controller/Author.pm +++ b/lib/MetaCPAN/Server/Controller/Author.pm @@ -64,8 +64,8 @@ sub by_users : Path('by_user') : Args(0) { # /author/by_prefix/PAUSE_ID_PREFIX sub by_prefix : Path('by_prefix') : Args(1) { my ( $self, $c, $prefix ) = @_; - my ($size) = $c->read_param('size')->[0] // 500; - my ($from) = $c->read_param('from')->[0] // 0; + my $size = $c->req->param('size') // 500; + my $from = $c->req->param('from') // 0; $c->stash_or_detach( $self->model($c) ->prefix_search( $prefix, { size => $size, from => $from } ) ); From 10f720c324d8db5ce42c865fc0a681a14a78e7c2 Mon Sep 17 00:00:00 2001 From: Leo Lapworth Date: Sun, 17 Jun 2018 18:25:19 +0100 Subject: [PATCH 023/892] do not rename restored indexes as ES::Model does not play nice --- lib/MetaCPAN/Script/Snapshot.pm | 20 +++++++++++++++----- 1 file changed, 15 insertions(+), 5 deletions(-) diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm index 160c658b8..ed8595366 100644 --- a/lib/MetaCPAN/Script/Snapshot.pm +++ b/lib/MetaCPAN/Script/Snapshot.pm @@ -121,7 +121,7 @@ has http_client => ( sub _build_http_client { return HTTP::Tiny->new( default_headers => { 'Accept' => 'application/json' }, - timeout => 120, # list can be slow + timeout => 120, # list can be slow ); } @@ -224,21 +224,28 @@ sub run_restore { my $snap_name = $self->snap_name; - $self->are_you_sure('Restoring... will rename indices to restored_XX'); + $self->are_you_sure( + 'Restoring... will NOT rename indices as ES::Model breaks'); + # IF we were not using ES::Model!.. # This is a safety feature, we can always # create aliases to point to them if required # just make sure there is enough disk space my $data = { - "rename_pattern" => '(.+)', - "rename_replacement" => 'restored_$1', + + # "rename_pattern" => '(.+)', + # "rename_replacement" => 'restored_$1', }; + # We wait until it's actually done! my $path = "${repository_name}/${snap_name}/_restore"; my $response = $self->_request( 'post', $path, $data ); - log_info { 'restoring: ' . $snap_name } if $response; + log_info { + 'restoring: ' . $snap_name . ' - see /_cat/recovery for progress' + } + if $response; return $response; } @@ -333,6 +340,8 @@ See status of snapshot... curl localhost:9200/_snapshot/our_backups/SNAP-NAME/_status + curl localhost:9200/_cat/recovery + Add an alias to the restored index curl -X POST 'localhost:9200/_aliases' -d ' @@ -352,4 +361,5 @@ You will need to run --setup on any box you wish to restore to You will need es_aws_s3_access_key and es_aws_s3_secret setup in your local metacpan_server_local.conf + =cut From 4af5e54302273cf773c3e8737b0c07dce0b0f308 Mon Sep 17 00:00:00 2001 From: Leo Lapworth Date: Sun, 17 Jun 2018 19:23:25 +0100 Subject: [PATCH 024/892] different version of tidy all fix! --- git/hooks/pre-commit | 4 ++-- lib/MetaCPAN/Script/Snapshot.pm | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/git/hooks/pre-commit b/git/hooks/pre-commit index 5854030e1..45fc7ca28 100755 --- a/git/hooks/pre-commit +++ b/git/hooks/pre-commit @@ -5,5 +5,5 @@ use warnings; # Hack to use carton's local::lib. use lib 'local/lib/perl5'; -use Code::TidyAll::Git::Precommit; -Code::TidyAll::Git::Precommit->check( no_stash => 1 ); +#use Code::TidyAll::Git::Precommit; +#Code::TidyAll::Git::Precommit->check( no_stash => 1 ); diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm index ed8595366..851e66e8e 100644 --- a/lib/MetaCPAN/Script/Snapshot.pm +++ b/lib/MetaCPAN/Script/Snapshot.pm @@ -121,7 +121,7 @@ has http_client => ( sub _build_http_client { return HTTP::Tiny->new( default_headers => { 'Accept' => 'application/json' }, - timeout => 120, # list can be slow + timeout => 120, # list can be slow ); } From 6b56d0eb2621b5fc998112d62de66ee3a3d81e1e Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Wed, 13 Jun 2018 10:10:48 -0500 Subject: [PATCH 025/892] clear author's user when disconnecting from PAUSE --- lib/MetaCPAN/Document/Author.pm | 5 +++-- lib/MetaCPAN/Model/User/Account.pm | 19 +++++++++++++++++++ lib/MetaCPAN/Server/Controller/User.pm | 11 +++++------ 3 files changed, 27 insertions(+), 8 deletions(-) diff --git a/lib/MetaCPAN/Document/Author.pm b/lib/MetaCPAN/Document/Author.pm index f312240d1..7a3e5db84 100644 --- a/lib/MetaCPAN/Document/Author.pm +++ b/lib/MetaCPAN/Document/Author.pm @@ -37,8 +37,9 @@ has pauseid => ( ); has user => ( - is => 'ro', - writer => '_set_user', + is => 'ro', + writer => '_set_user', + clearer => '_clear_user', ); has gravatar_url => ( diff --git a/lib/MetaCPAN/Model/User/Account.pm b/lib/MetaCPAN/Model/User/Account.pm index af29ad84e..d130f5074 100644 --- a/lib/MetaCPAN/Model/User/Account.pm +++ b/lib/MetaCPAN/Model/User/Account.pm @@ -146,5 +146,24 @@ sub get_identities { return grep { $_->name eq $identity } @{ $self->identity }; } +sub remove_identity { + my ( $self, $identity ) = @_; + my $ids = $self->identities; + my ($id) = grep { $_->{name} eq $identity } @$ids; + @$ids = grep { $_->{name} ne $identity } @$ids; + + if ( $identity eq 'pause' ) { + my $profile = $self->index->model->index('cpan')->type('author') + ->get( $id->{key} ); + + if ( $profile && $profile->user eq $self->id ) { + $profile->_clear_user; + $profile->put; + } + } + + return $id; +} + __PACKAGE__->meta->make_immutable; 1; diff --git a/lib/MetaCPAN/Server/Controller/User.pm b/lib/MetaCPAN/Server/Controller/User.pm index b70c411d2..7f913b089 100644 --- a/lib/MetaCPAN/Server/Controller/User.pm +++ b/lib/MetaCPAN/Server/Controller/User.pm @@ -54,12 +54,11 @@ sub identity_GET { sub identity_DELETE { my ( $self, $c ) = @_; my ($identity) = @{ $c->req->arguments }; - my $ids = $c->user->identity; - ($identity) = grep { $_->name eq $identity } @$ids; - if ($identity) { - @$ids = grep { $_->{name} ne $identity->name } @$ids; - $c->user->put( { refresh => 1 } ); - $self->status_ok( $c, entity => $identity ); + my $user = $c->user; + if ( $user->has_identity($identity) ) { + my $id = $user->remove_identity($identity); + $user->put( { refresh => 1 } ); + $self->status_ok( $c, entity => $id ); } else { $self->status_not_found( $c, message => 'Identity doesn\'t exist' ); From 3dce8ffab3cbf536c3d0de79bc82a15805a6a518 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Wed, 20 Jun 2018 23:12:58 +0100 Subject: [PATCH 026/892] Canonize the path provided by META to match dir listing When parsing a release we compare the paths of files in 'provides' with the directory listing. When these entries in META contain './' they don't match the listing. This causes the 'modules' section in the 'release' entry to miss files and in turn (if 'modules' ends up empty) the release cannot be marked as 'latest'. --- lib/MetaCPAN/Model/Release.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 5a03d645f..93e5064b4 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -8,6 +8,7 @@ use CPAN::DistnameInfo (); use CPAN::Meta (); use DateTime (); use File::Find (); +use File::Spec (); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Model::Archive; use MetaCPAN::Types qw(ArrayRef AbsFile Str); @@ -485,7 +486,7 @@ sub _modules_from_meta { my $files = $self->files; foreach my $module ( sort keys %$provides ) { my $data = $provides->{$module}; - my $path = $data->{file}; + my $path = File::Spec->canonpath( $data->{file} ); # Obey no_index and take the shortest path if multiple files match. my ($file) = sort { length( $a->path ) <=> length( $b->path ) } From 95a41c4975f1dc2ada1f38f0d091a84125ec6003 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 21 Jun 2018 13:55:40 -0400 Subject: [PATCH 027/892] Revert "different version of tidy all fix!" This reverts commit 4af5e54302273cf773c3e8737b0c07dce0b0f308. --- git/hooks/pre-commit | 4 ++-- lib/MetaCPAN/Script/Snapshot.pm | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/git/hooks/pre-commit b/git/hooks/pre-commit index 45fc7ca28..5854030e1 100755 --- a/git/hooks/pre-commit +++ b/git/hooks/pre-commit @@ -5,5 +5,5 @@ use warnings; # Hack to use carton's local::lib. use lib 'local/lib/perl5'; -#use Code::TidyAll::Git::Precommit; -#Code::TidyAll::Git::Precommit->check( no_stash => 1 ); +use Code::TidyAll::Git::Precommit; +Code::TidyAll::Git::Precommit->check( no_stash => 1 ); diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm index 851e66e8e..ed8595366 100644 --- a/lib/MetaCPAN/Script/Snapshot.pm +++ b/lib/MetaCPAN/Script/Snapshot.pm @@ -121,7 +121,7 @@ has http_client => ( sub _build_http_client { return HTTP::Tiny->new( default_headers => { 'Accept' => 'application/json' }, - timeout => 120, # list can be slow + timeout => 120, # list can be slow ); } From 0fbf5cf57de9a218b47925bece432fa76e8fe178 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 21 Jun 2018 13:56:10 -0400 Subject: [PATCH 028/892] Tidy --- lib/MetaCPAN/Script/Snapshot.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm index ed8595366..851e66e8e 100644 --- a/lib/MetaCPAN/Script/Snapshot.pm +++ b/lib/MetaCPAN/Script/Snapshot.pm @@ -121,7 +121,7 @@ has http_client => ( sub _build_http_client { return HTTP::Tiny->new( default_headers => { 'Accept' => 'application/json' }, - timeout => 120, # list can be slow + timeout => 120, # list can be slow ); } From 90bc452b320bdf0956eb24f6508854ddc6ff9eca Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sun, 22 Apr 2018 09:22:52 +0100 Subject: [PATCH 029/892] Script code updates Specifically, the main change here is in the Latest script, to allow easier debugging/introspection of issues. The query is set into a variable (to allow dumping) and the $data variable was renamed as the name was identified as a confusing one. --- lib/MetaCPAN/Script/Latest.pm | 80 +++++++++++++++++++--------------- lib/MetaCPAN/Script/Mapping.pm | 20 ++++----- lib/MetaCPAN/Script/Session.pm | 2 +- 3 files changed, 55 insertions(+), 47 deletions(-) diff --git a/lib/MetaCPAN/Script/Latest.pm b/lib/MetaCPAN/Script/Latest.pm index 40ddd9c88..ca7bca55e 100644 --- a/lib/MetaCPAN/Script/Latest.pm +++ b/lib/MetaCPAN/Script/Latest.pm @@ -87,25 +87,30 @@ sub run { ? { terms => { "module.name" => \@filter } } : { exists => { field => "module.name" } }; - my $scroll = $self->index->type('file')->filter( - { - bool => { - must => [ - { - nested => { - path => 'module', - filter => { bool => { must => \@module_filters } } - } - }, - { term => { 'maturity' => 'released' } }, - ], - must_not => [ - { term => { status => 'backpan' } }, - { term => { distribution => 'perl' } } - ] - } + # This query will be used to produce a (scrolled) list of + # 'file' type records where the module.name matches the + # distribution name and which are released & + # indexed (the 'leading' module) + my $query = { + bool => { + must => [ + { + nested => { + path => 'module', + filter => { bool => { must => \@module_filters } } + } + }, + { term => { 'maturity' => 'released' } }, + ], + must_not => [ + { term => { status => 'backpan' } }, + { term => { distribution => 'perl' } } + ] } - ) + }; + + my $scroll + = $self->index->type('file')->filter($query) ->source( [qw< author date distribution module.name release status >] ) ->size(100)->raw->scroll; @@ -122,13 +127,13 @@ sub run { while ( my $file = $scroll->next ) { $i++; log_debug { "$i of " . $scroll->total } unless ( $i % 1000 ); - my $data = $file->{_source}; + my $file_data = $file->{_source}; # Convert module name into Parse::CPAN::Packages::Fast::Package object. my @modules = grep {defined} map { eval { $p->package( $_->{name} ) } - } @{ $data->{module} }; + } @{ $file_data->{module} }; push @modules_to_purge, @modules; @@ -152,21 +157,24 @@ sub run { # (like /\.pm\.gz$/) so distvname might not be present. # I assume cpanid always will be. if ( defined( $dist->distvname ) - && $dist->distvname eq $data->{release} - && $dist->cpanid eq $data->{author} ) + && $dist->distvname eq $file_data->{release} + && $dist->cpanid eq $file_data->{author} ) { - my $upgrade = $upgrade{ $data->{distribution} }; + my $upgrade = $upgrade{ $file_data->{distribution} }; # If multiple versions of a dist appear in 02packages # only mark the most recent upload as latest. next - if ( $upgrade - && $self->compare_dates( $upgrade->{date}, $data->{date} ) + if ( + $upgrade + && $self->compare_dates( + $upgrade->{date}, $file_data->{date} + ) ); - $upgrade{ $data->{distribution} } = $data; + $upgrade{ $file_data->{distribution} } = $file_data; } - elsif ( $data->{status} eq 'latest' ) { - $downgrade{ $data->{release} } = $data; + elsif ( $file_data->{status} eq 'latest' ) { + $downgrade{ $file_data->{release} } = $file_data; } } } @@ -176,16 +184,16 @@ sub run { type => 'file' ); - while ( my ( $dist, $data ) = each %upgrade ) { + while ( my ( $dist, $file_data ) = each %upgrade ) { # Don't reindex if already marked as latest. # This just means that it hasn't changed (query includes 'latest'). - next if ( !$self->force and $data->{status} eq 'latest' ); + next if ( !$self->force and $file_data->{status} eq 'latest' ); - $self->reindex( $bulk, $data, 'latest' ); + $self->reindex( $bulk, $file_data, 'latest' ); } - while ( my ( $release, $data ) = each %downgrade ) { + while ( my ( $release, $file_data ) = each %downgrade ) { # Don't downgrade if this release version is also marked as latest. # This could happen if a module is moved to a new dist @@ -193,11 +201,11 @@ sub run { # This could also include bug fixes in our indexer, PAUSE, etc. next if ( !$self->force - && $upgrade{ $data->{distribution} } - && $upgrade{ $data->{distribution} }->{release} eq - $data->{release} ); + && $upgrade{ $file_data->{distribution} } + && $upgrade{ $file_data->{distribution} }->{release} eq + $file_data->{release} ); - $self->reindex( $bulk, $data, 'cpan' ); + $self->reindex( $bulk, $file_data, 'cpan' ); } $bulk->flush; $self->index->refresh; diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index 51291134d..dffb75254 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -297,7 +297,7 @@ sub copy_type { sub _copy_slice { my ( $self, $query, $index, $type ) = @_; - my $scroll = $self->es()->scroll_helper( + my $scroll = $self->es->scroll_helper( search_type => 'scan', size => 250, scroll => '10m', @@ -341,7 +341,7 @@ sub empty_type { max_count => 500, ); - my $scroll = $self->es()->scroll_helper( + my $scroll = $self->es->scroll_helper( search_type => 'scan', size => 250, scroll => '10m', @@ -381,37 +381,37 @@ sub deploy_mapping { author => decode_json(MetaCPAN::Script::Mapping::CPAN::Author::mapping), distribution => - decode_json(MetaCPAN::Script::Mapping::CPAN::Distribution::mapping + decode_json( MetaCPAN::Script::Mapping::CPAN::Distribution::mapping ), favorite => - decode_json(MetaCPAN::Script::Mapping::CPAN::Favorite::mapping + decode_json( MetaCPAN::Script::Mapping::CPAN::Favorite::mapping ), file => decode_json(MetaCPAN::Script::Mapping::CPAN::File::mapping), mirror => decode_json(MetaCPAN::Script::Mapping::CPAN::Mirror::mapping), permission => - decode_json(MetaCPAN::Script::Mapping::CPAN::Permission::mapping + decode_json( MetaCPAN::Script::Mapping::CPAN::Permission::mapping ), package => - decode_json(MetaCPAN::Script::Mapping::CPAN::Package::mapping + decode_json( MetaCPAN::Script::Mapping::CPAN::Package::mapping ), rating => decode_json(MetaCPAN::Script::Mapping::CPAN::Rating::mapping), release => - decode_json(MetaCPAN::Script::Mapping::CPAN::Release::mapping + decode_json( MetaCPAN::Script::Mapping::CPAN::Release::mapping ), }, user => { account => - decode_json(MetaCPAN::Script::Mapping::User::Account::mapping + decode_json( MetaCPAN::Script::Mapping::User::Account::mapping ), identity => - decode_json(MetaCPAN::Script::Mapping::User::Identity::mapping + decode_json( MetaCPAN::Script::Mapping::User::Identity::mapping ), session => - decode_json(MetaCPAN::Script::Mapping::User::Session::mapping + decode_json( MetaCPAN::Script::Mapping::User::Session::mapping ), }, contributor => { diff --git a/lib/MetaCPAN/Script/Session.pm b/lib/MetaCPAN/Script/Session.pm index 503aaa490..3dd6c7ad2 100644 --- a/lib/MetaCPAN/Script/Session.pm +++ b/lib/MetaCPAN/Script/Session.pm @@ -11,7 +11,7 @@ with 'MetaCPAN::Role::Script', 'MooseX::Getopt'; sub run { my $self = shift; - my $scroll = $self->es()->scroll_helper( + my $scroll = $self->es->scroll_helper( size => 10_000, scroll => '1m', index => 'user', From c6912740d490062ccdfa6ca1e1e96ddbb557c849 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sun, 22 Apr 2018 09:46:41 +0100 Subject: [PATCH 030/892] Mapping script: bug fix When creating a new index, we don't need to re-apply the mappings for the main index (this just leads to warnings) --- lib/MetaCPAN/Script/Mapping.pm | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/lib/MetaCPAN/Script/Mapping.pm b/lib/MetaCPAN/Script/Mapping.pm index dffb75254..65e819f45 100644 --- a/lib/MetaCPAN/Script/Mapping.pm +++ b/lib/MetaCPAN/Script/Mapping.pm @@ -198,11 +198,11 @@ sub create_index { my $dst_idx = $self->arg_create_index; $self->_check_index_exists( $dst_idx, NOT_EXPECTED ); - my $patch_mapping = decode_json $self->patch_mapping; - my @patch_types = sort keys %{$patch_mapping}; - my $dep = $self->index->deployment_statement; - my $existing_mapping = delete $dep->{mappings}; - my $mapping = $self->skip_existing_mapping ? +{} : $existing_mapping; + my $patch_mapping = decode_json $self->patch_mapping; + my @patch_types = sort keys %{$patch_mapping}; + my $dep = $self->index->deployment_statement; + delete $dep->{mappings}; + my $mapping = +{}; # create the new index with the copied settings log_info {"Creating index: $dst_idx"}; @@ -381,37 +381,37 @@ sub deploy_mapping { author => decode_json(MetaCPAN::Script::Mapping::CPAN::Author::mapping), distribution => - decode_json( MetaCPAN::Script::Mapping::CPAN::Distribution::mapping + decode_json(MetaCPAN::Script::Mapping::CPAN::Distribution::mapping ), favorite => - decode_json( MetaCPAN::Script::Mapping::CPAN::Favorite::mapping + decode_json(MetaCPAN::Script::Mapping::CPAN::Favorite::mapping ), file => decode_json(MetaCPAN::Script::Mapping::CPAN::File::mapping), mirror => decode_json(MetaCPAN::Script::Mapping::CPAN::Mirror::mapping), permission => - decode_json( MetaCPAN::Script::Mapping::CPAN::Permission::mapping + decode_json(MetaCPAN::Script::Mapping::CPAN::Permission::mapping ), package => - decode_json( MetaCPAN::Script::Mapping::CPAN::Package::mapping + decode_json(MetaCPAN::Script::Mapping::CPAN::Package::mapping ), rating => decode_json(MetaCPAN::Script::Mapping::CPAN::Rating::mapping), release => - decode_json( MetaCPAN::Script::Mapping::CPAN::Release::mapping + decode_json(MetaCPAN::Script::Mapping::CPAN::Release::mapping ), }, user => { account => - decode_json( MetaCPAN::Script::Mapping::User::Account::mapping + decode_json(MetaCPAN::Script::Mapping::User::Account::mapping ), identity => - decode_json( MetaCPAN::Script::Mapping::User::Identity::mapping + decode_json(MetaCPAN::Script::Mapping::User::Identity::mapping ), session => - decode_json( MetaCPAN::Script::Mapping::User::Session::mapping + decode_json(MetaCPAN::Script::Mapping::User::Session::mapping ), }, contributor => { From 683da7244f527027e7ec6023feabc6267244a0b4 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 16 Jun 2018 13:27:50 -0500 Subject: [PATCH 031/892] use new Search::Elasticsearch and ElasticSearchX::Model --- cpanfile | 7 +++++-- lib/MetaCPAN/Script/CPANTesters.pm | 3 ++- lib/MetaCPAN/Script/CPANTestersAPI.pm | 3 ++- lib/MetaCPAN/Server/Model/CPAN.pm | 4 ++-- metacpan_server.conf | 24 ++++++++++++++++++++++++ metacpan_server_testing.conf | 12 +++++++++--- t/lib/MetaCPAN/TestServer.pm | 14 ++++++++------ 7 files changed, 52 insertions(+), 15 deletions(-) diff --git a/cpanfile b/cpanfile index 004e3dc51..b4022f96e 100644 --- a/cpanfile +++ b/cpanfile @@ -45,7 +45,7 @@ requires 'Devel::ArgNames'; requires 'Digest::MD5'; requires 'Digest::SHA'; requires 'EV'; -requires 'ElasticSearchX::Model', '1.0.2'; +requires 'ElasticSearchX::Model', '2.0.0'; requires 'Email::Address'; requires 'Email::Sender::Simple'; requires 'Email::Simple'; @@ -74,6 +74,7 @@ requires 'HTML::Entities'; requires 'HTML::TokeParser::Simple'; requires 'HTTP::Request::Common'; requires 'Hash::Merge::Simple'; +requires 'Hijk' => '0.27'; requires 'IO::All'; requires 'IO::Interactive'; requires 'IO::Prompt'; @@ -156,7 +157,9 @@ requires 'Ref::Util'; requires 'Regexp::Common'; requires 'Regexp::Common::time'; requires 'Safe', '2.35'; # bug fixes (used by Parse::PMFile) -requires 'Search::Elasticsearch', '== 2.03'; +requires 'Search::Elasticsearch' => '6.00'; +requires 'Search::Elasticsearch::Client::2_0::Direct' => '5.02'; +requires 'Search::Elasticsearch::Cxn::Hijk' => '6.00'; requires 'Starman'; requires 'Throwable::Error'; requires 'Time::Local'; diff --git a/lib/MetaCPAN/Script/CPANTesters.pm b/lib/MetaCPAN/Script/CPANTesters.pm index f49120dc5..498d4f114 100644 --- a/lib/MetaCPAN/Script/CPANTesters.pm +++ b/lib/MetaCPAN/Script/CPANTesters.pm @@ -10,6 +10,7 @@ use File::stat qw(stat); use IO::Uncompress::Bunzip2 qw(bunzip2); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types qw( Bool File Uri ); +use ElasticSearchX::Model::Document::Types qw(ESBulk); use Moose; with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; @@ -46,7 +47,7 @@ has skip_download => ( has _bulk => ( is => 'ro', - isa => 'Search::Elasticsearch::Bulk', + isa => ESBulk, lazy => 1, default => sub { $_[0]->es->bulk_helper( diff --git a/lib/MetaCPAN/Script/CPANTestersAPI.pm b/lib/MetaCPAN/Script/CPANTestersAPI.pm index a2ee0b094..b4082521f 100644 --- a/lib/MetaCPAN/Script/CPANTestersAPI.pm +++ b/lib/MetaCPAN/Script/CPANTestersAPI.pm @@ -6,6 +6,7 @@ use warnings; use Log::Contextual qw( :log :dlog ); use Cpanel::JSON::XS qw( decode_json ); use MetaCPAN::Types qw( Uri ); +use ElasticSearchX::Model::Document::Types qw(ESBulk); use Moose; with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; @@ -28,7 +29,7 @@ sub _build_url { has _bulk => ( is => 'ro', - isa => 'Search::Elasticsearch::Bulk', + isa => ESBulk, lazy => 1, default => sub { $_[0]->es->bulk_helper( diff --git a/lib/MetaCPAN/Server/Model/CPAN.pm b/lib/MetaCPAN/Server/Model/CPAN.pm index ae547e23f..0e8196fa6 100644 --- a/lib/MetaCPAN/Server/Model/CPAN.pm +++ b/lib/MetaCPAN/Server/Model/CPAN.pm @@ -22,13 +22,13 @@ has index => ( default => 'cpan', ); -has servers => ( +has es_config => ( is => 'ro', default => ':9200', ); sub _build_esx_model { - MetaCPAN::Model->new( es => shift->servers ); + MetaCPAN::Model->new( es => shift->es_config ); } sub type { diff --git a/metacpan_server.conf b/metacpan_server.conf index 5558ea850..ca40a56de 100644 --- a/metacpan_server.conf +++ b/metacpan_server.conf @@ -10,3 +10,27 @@ minion_dsn = postgresql:///minion_queue secret_key 8225b1874fdc431cedb1cf7d454a92b8fde3a5e6 + + + + nodes localhost:9200 + client 2_0::Direct + cxn Hijk + + + + + + nodes localhost:9200 + client 2_0::Direct + cxn Hijk + + + + + + nodes localhost:9200 + client 2_0::Direct + cxn Hijk + + diff --git a/metacpan_server_testing.conf b/metacpan_server_testing.conf index 59e200d54..173aca5c8 100644 --- a/metacpan_server_testing.conf +++ b/metacpan_server_testing.conf @@ -2,15 +2,21 @@ cpan var/t/tmp/fakecpan source_base var/t/tmp/source - servers __ENV(ES)__ + + nodes __ENV(ES)__ + - servers __ENV(ES)__ + + nodes __ENV(ES)__ + - servers __ENV(ES)__ + + nodes __ENV(ES)__ + diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index 5c814a017..bcbc8d04d 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -103,11 +103,12 @@ sub _build_es_server { my $self = shift; my $server = Search::Elasticsearch::TestServer->new( - conf => [ 'cluster.name' => 'metacpan-test' ], - es_home => $self->_es_home, - es_port => 9700, - http_port => 9900, - instances => 1, + conf => [ 'cluster.name' => 'metacpan-test' ], + es_home => $self->_es_home, + es_port => 9700, + http_port => 9900, + instances => 1, + es_version => '2_0', ); diag 'Connecting to Elasticsearch on ' . $self->_es_home; @@ -139,7 +140,8 @@ sub _build_es_client { my $es = Search::Elasticsearch->new( nodes => $self->_es_home, - ( $ENV{ES_TRACE} ? ( trace_to => [ 'File', 'es.log' ] ) : () ) + ( $ENV{ES_TRACE} ? ( trace_to => [ 'File', 'es.log' ] ) : () ), + client => '2_0::Direct', ); ok( $es, 'got ElasticSearch object' ); From 6fb73c60df4c96c5e4e77645a3df73464994c6f0 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 25 Jun 2018 14:50:11 +0100 Subject: [PATCH 032/892] update cpanfile.snapshot --- cpanfile.snapshot | 208 ++++++++++++++++++++++++++-------------------- 1 file changed, 119 insertions(+), 89 deletions(-) diff --git a/cpanfile.snapshot b/cpanfile.snapshot index db1763ca7..2794c565a 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -2459,32 +2459,32 @@ DISTRIBUTIONS Canary::Stability 0 ExtUtils::MakeMaker 6.52 common::sense 0 - ElasticSearchX-Model-1.0.2 - pathname: O/OA/OALDERS/ElasticSearchX-Model-1.0.2.tar.gz - provides: - ElasticSearchX::Model v1.0.2 - ElasticSearchX::Model::Bulk v1.0.2 - ElasticSearchX::Model::Document v1.0.2 - ElasticSearchX::Model::Document::EmbeddedRole v1.0.2 - ElasticSearchX::Model::Document::Mapping v1.0.2 - ElasticSearchX::Model::Document::Role v1.0.2 - ElasticSearchX::Model::Document::Set v1.0.2 - ElasticSearchX::Model::Document::Trait::Attribute v1.0.2 - ElasticSearchX::Model::Document::Trait::Class v1.0.2 - ElasticSearchX::Model::Document::Trait::Class::ID v1.0.2 - ElasticSearchX::Model::Document::Trait::Class::Timestamp v1.0.2 - ElasticSearchX::Model::Document::Trait::Class::Version v1.0.2 - ElasticSearchX::Model::Document::Trait::Field::ID v1.0.2 - ElasticSearchX::Model::Document::Trait::Field::TTL v1.0.2 - ElasticSearchX::Model::Document::Trait::Field::Timestamp v1.0.2 - ElasticSearchX::Model::Document::Trait::Field::Version v1.0.2 - ElasticSearchX::Model::Document::Types v1.0.2 - ElasticSearchX::Model::Index v1.0.2 - ElasticSearchX::Model::Role v1.0.2 - ElasticSearchX::Model::Scroll v1.0.2 - ElasticSearchX::Model::Trait::Class v1.0.2 - ElasticSearchX::Model::Tutorial v1.0.2 - ElasticSearchX::Model::Util v1.0.2 + ElasticSearchX-Model-2.0.0 + pathname: O/OA/OALDERS/ElasticSearchX-Model-2.0.0.tar.gz + provides: + ElasticSearchX::Model v2.0.0 + ElasticSearchX::Model::Bulk v2.0.0 + ElasticSearchX::Model::Document v2.0.0 + ElasticSearchX::Model::Document::EmbeddedRole v2.0.0 + ElasticSearchX::Model::Document::Mapping v2.0.0 + ElasticSearchX::Model::Document::Role v2.0.0 + ElasticSearchX::Model::Document::Set v2.0.0 + ElasticSearchX::Model::Document::Trait::Attribute v2.0.0 + ElasticSearchX::Model::Document::Trait::Class v2.0.0 + ElasticSearchX::Model::Document::Trait::Class::ID v2.0.0 + ElasticSearchX::Model::Document::Trait::Class::Timestamp v2.0.0 + ElasticSearchX::Model::Document::Trait::Class::Version v2.0.0 + ElasticSearchX::Model::Document::Trait::Field::ID v2.0.0 + ElasticSearchX::Model::Document::Trait::Field::TTL v2.0.0 + ElasticSearchX::Model::Document::Trait::Field::Timestamp v2.0.0 + ElasticSearchX::Model::Document::Trait::Field::Version v2.0.0 + ElasticSearchX::Model::Document::Types v2.0.0 + ElasticSearchX::Model::Index v2.0.0 + ElasticSearchX::Model::Role v2.0.0 + ElasticSearchX::Model::Scroll v2.0.0 + ElasticSearchX::Model::Trait::Class v2.0.0 + ElasticSearchX::Model::Tutorial v2.0.0 + ElasticSearchX::Model::Util v2.0.0 requirements: Carp 0 Class::Load 0 @@ -2493,10 +2493,10 @@ DISTRIBUTIONS DateTime::Format::ISO8601 0 Digest::SHA1 0 Eval::Closure 0 + ExtUtils::MakeMaker 0 JSON::MaybeXS 0 List::MoreUtils 0 List::Util 0 - Module::Build 0.3601 Module::Find 0 Moose 2.02 Moose::Exporter 0 @@ -2513,9 +2513,8 @@ DISTRIBUTIONS MooseX::Types::Structured 0 Scalar::Util 0 Search::Elasticsearch 2.02 - Search::Elasticsearch::Bulk 0 - Search::Elasticsearch::Scroll 0 Sub::Exporter 0 + perl 5.006 strict 0 version 0 warnings 0 @@ -3603,6 +3602,14 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.008001 + Hijk-0.27 + pathname: G/GU/GUGOD/Hijk-0.27.tar.gz + provides: + Hijk 0.27 + requirements: + CPAN::Meta 0 + ExtUtils::MakeMaker 6.36 + Time::HiRes 0 Hook-LexWrap-0.26 pathname: E/ET/ETHER/Hook-LexWrap-0.26.tar.gz provides: @@ -7547,65 +7554,54 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Test::More 0 perl 5.006001 - Search-Elasticsearch-2.03 - pathname: D/DR/DRTECH/Search-Elasticsearch-2.03.tar.gz - provides: - Search::Elasticsearch 2.03 - Search::Elasticsearch::Bulk 2.03 - Search::Elasticsearch::Client::0_90::Direct 2.03 - Search::Elasticsearch::Client::0_90::Direct::Cluster 2.03 - Search::Elasticsearch::Client::0_90::Direct::Indices 2.03 - Search::Elasticsearch::Client::1_0::Direct 2.03 - Search::Elasticsearch::Client::1_0::Direct::Cat 2.03 - Search::Elasticsearch::Client::1_0::Direct::Cluster 2.03 - Search::Elasticsearch::Client::1_0::Direct::Indices 2.03 - Search::Elasticsearch::Client::1_0::Direct::Nodes 2.03 - Search::Elasticsearch::Client::1_0::Direct::Snapshot 2.03 - Search::Elasticsearch::Client::2_0::Direct 2.03 - Search::Elasticsearch::Client::2_0::Direct::Cat 2.03 - Search::Elasticsearch::Client::2_0::Direct::Cluster 2.03 - Search::Elasticsearch::Client::2_0::Direct::Indices 2.03 - Search::Elasticsearch::Client::2_0::Direct::Nodes 2.03 - Search::Elasticsearch::Client::2_0::Direct::Snapshot 2.03 - Search::Elasticsearch::Client::2_0::Direct::Tasks 2.03 - Search::Elasticsearch::Cxn::Factory 2.03 - Search::Elasticsearch::Cxn::HTTPTiny 2.03 - Search::Elasticsearch::Cxn::Hijk 2.03 - Search::Elasticsearch::Cxn::LWP 2.03 - Search::Elasticsearch::CxnPool::Sniff 2.03 - Search::Elasticsearch::CxnPool::Static 2.03 - Search::Elasticsearch::CxnPool::Static::NoPing 2.03 - Search::Elasticsearch::Error 2.03 - Search::Elasticsearch::Logger::LogAny 2.03 - Search::Elasticsearch::Role::API::0_90 2.03 - Search::Elasticsearch::Role::API::1_0 2.03 - Search::Elasticsearch::Role::API::2_0 2.03 - Search::Elasticsearch::Role::Bulk 2.03 - Search::Elasticsearch::Role::Client 2.03 - Search::Elasticsearch::Role::Client::Direct 2.03 - Search::Elasticsearch::Role::Client::Direct::Main 2.03 - Search::Elasticsearch::Role::Cxn 2.03 - Search::Elasticsearch::Role::Cxn::HTTP 2.03 - Search::Elasticsearch::Role::CxnPool 2.03 - Search::Elasticsearch::Role::CxnPool::Sniff 2.03 - Search::Elasticsearch::Role::CxnPool::Static 2.03 - Search::Elasticsearch::Role::CxnPool::Static::NoPing 2.03 - Search::Elasticsearch::Role::Is_Sync 2.03 - Search::Elasticsearch::Role::Logger 2.03 - Search::Elasticsearch::Role::Scroll 2.03 - Search::Elasticsearch::Role::Serializer 2.03 - Search::Elasticsearch::Role::Serializer::JSON 2.03 - Search::Elasticsearch::Role::Transport 2.03 - Search::Elasticsearch::Scroll 2.03 - Search::Elasticsearch::Serializer::JSON 2.03 - Search::Elasticsearch::Serializer::JSON::Cpanel 2.03 - Search::Elasticsearch::Serializer::JSON::PP 2.03 - Search::Elasticsearch::Serializer::JSON::XS 2.03 - Search::Elasticsearch::TestServer 2.03 - Search::Elasticsearch::Transport 2.03 - Search::Elasticsearch::Util 2.03 - Search::Elasticsearch::Util::API::Path 2.03 - Search::Elasticsearch::Util::API::QS 2.03 + Search-Elasticsearch-6.00 + pathname: D/DR/DRTECH/Search-Elasticsearch-6.00.tar.gz + provides: + Search::Elasticsearch 6.00 + Search::Elasticsearch::Client::6_0 6.00 + Search::Elasticsearch::Client::6_0::Bulk 6.00 + Search::Elasticsearch::Client::6_0::Direct 6.00 + Search::Elasticsearch::Client::6_0::Direct::Cat 6.00 + Search::Elasticsearch::Client::6_0::Direct::Cluster 6.00 + Search::Elasticsearch::Client::6_0::Direct::Indices 6.00 + Search::Elasticsearch::Client::6_0::Direct::Ingest 6.00 + Search::Elasticsearch::Client::6_0::Direct::Nodes 6.00 + Search::Elasticsearch::Client::6_0::Direct::Snapshot 6.00 + Search::Elasticsearch::Client::6_0::Direct::Tasks 6.00 + Search::Elasticsearch::Client::6_0::Role::API 6.00 + Search::Elasticsearch::Client::6_0::Role::Bulk 6.00 + Search::Elasticsearch::Client::6_0::Role::Scroll 6.00 + Search::Elasticsearch::Client::6_0::Scroll 6.00 + Search::Elasticsearch::Client::6_0::TestServer 6.00 + Search::Elasticsearch::Cxn::Factory 6.00 + Search::Elasticsearch::Cxn::HTTPTiny 6.00 + Search::Elasticsearch::Cxn::Hijk 6.00 + Search::Elasticsearch::Cxn::LWP 6.00 + Search::Elasticsearch::CxnPool::Sniff 6.00 + Search::Elasticsearch::CxnPool::Static 6.00 + Search::Elasticsearch::CxnPool::Static::NoPing 6.00 + Search::Elasticsearch::Error 6.00 + Search::Elasticsearch::Logger::LogAny 6.00 + Search::Elasticsearch::Role::API 6.00 + Search::Elasticsearch::Role::Client 6.00 + Search::Elasticsearch::Role::Client::Direct 6.00 + Search::Elasticsearch::Role::Cxn 6.00 + Search::Elasticsearch::Role::CxnPool 6.00 + Search::Elasticsearch::Role::CxnPool::Sniff 6.00 + Search::Elasticsearch::Role::CxnPool::Static 6.00 + Search::Elasticsearch::Role::CxnPool::Static::NoPing 6.00 + Search::Elasticsearch::Role::Is_Sync 6.00 + Search::Elasticsearch::Role::Logger 6.00 + Search::Elasticsearch::Role::Serializer 6.00 + Search::Elasticsearch::Role::Serializer::JSON 6.00 + Search::Elasticsearch::Role::Transport 6.00 + Search::Elasticsearch::Serializer::JSON 6.00 + Search::Elasticsearch::Serializer::JSON::Cpanel 6.00 + Search::Elasticsearch::Serializer::JSON::PP 6.00 + Search::Elasticsearch::Serializer::JSON::XS 6.00 + Search::Elasticsearch::TestServer 6.00 + Search::Elasticsearch::Transport 6.00 + Search::Elasticsearch::Util 6.00 requirements: Any::URI::Escape 0 Data::Dumper 0 @@ -7616,8 +7612,11 @@ DISTRIBUTIONS HTTP::Headers 0 HTTP::Request 0 HTTP::Tiny 0.043 + IO::Compress::Deflate 0 + IO::Compress::Gzip 0 IO::Select 0 IO::Socket 0 + IO::Uncompress::Gunzip 0 IO::Uncompress::Inflate 0 JSON::MaybeXS 1.002002 JSON::PP 0 @@ -7627,7 +7626,7 @@ DISTRIBUTIONS Log::Any::Adapter 0 MIME::Base64 0 Module::Runtime 0 - Moo 1.003 + Moo 2.001000 Moo::Role 0 POSIX 0 Package::Stash 0.34 @@ -7640,6 +7639,37 @@ DISTRIBUTIONS overload 0 strict 0 warnings 0 + Search-Elasticsearch-Client-2_0-5.02 + pathname: D/DR/DRTECH/Search-Elasticsearch-Client-2_0-5.02.tar.gz + provides: + Search::Elasticsearch::Client::2_0 5.02 + Search::Elasticsearch::Client::2_0::Bulk 5.02 + Search::Elasticsearch::Client::2_0::Direct 5.02 + Search::Elasticsearch::Client::2_0::Direct::Cat 5.02 + Search::Elasticsearch::Client::2_0::Direct::Cluster 5.02 + Search::Elasticsearch::Client::2_0::Direct::Indices 5.02 + Search::Elasticsearch::Client::2_0::Direct::Nodes 5.02 + Search::Elasticsearch::Client::2_0::Direct::Snapshot 5.02 + Search::Elasticsearch::Client::2_0::Direct::Tasks 5.02 + Search::Elasticsearch::Client::2_0::Role::API 5.02 + Search::Elasticsearch::Client::2_0::Role::Bulk 5.02 + Search::Elasticsearch::Client::2_0::Role::Scroll 5.02 + Search::Elasticsearch::Client::2_0::Scroll 5.02 + Search::Elasticsearch::Client::2_0::TestServer 5.02 + requirements: + Devel::GlobalDestruction 0 + ExtUtils::MakeMaker 0 + Moo 0 + Moo::Role 0 + Search::Elasticsearch 5.02 + Search::Elasticsearch::Role::API 0 + Search::Elasticsearch::Role::Client::Direct 0 + Search::Elasticsearch::Role::Is_Sync 0 + Search::Elasticsearch::Util 0 + Try::Tiny 0 + namespace::clean 0 + strict 0 + warnings 0 Server-Starter-0.33 pathname: K/KA/KAZUHO/Server-Starter-0.33.tar.gz provides: From e70f90c0c1871e625e7cba7f3ecba8be699e437a Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 28 Jun 2018 00:33:46 +0200 Subject: [PATCH 033/892] Revert "use new Search::Elasticsearch and ElasticSearchX::Model" This reverts commit 683da7244f527027e7ec6023feabc6267244a0b4. --- cpanfile | 7 ++----- lib/MetaCPAN/Script/CPANTesters.pm | 3 +-- lib/MetaCPAN/Script/CPANTestersAPI.pm | 3 +-- lib/MetaCPAN/Server/Model/CPAN.pm | 4 ++-- metacpan_server.conf | 24 ------------------------ metacpan_server_testing.conf | 12 +++--------- t/lib/MetaCPAN/TestServer.pm | 14 ++++++-------- 7 files changed, 15 insertions(+), 52 deletions(-) diff --git a/cpanfile b/cpanfile index b4022f96e..004e3dc51 100644 --- a/cpanfile +++ b/cpanfile @@ -45,7 +45,7 @@ requires 'Devel::ArgNames'; requires 'Digest::MD5'; requires 'Digest::SHA'; requires 'EV'; -requires 'ElasticSearchX::Model', '2.0.0'; +requires 'ElasticSearchX::Model', '1.0.2'; requires 'Email::Address'; requires 'Email::Sender::Simple'; requires 'Email::Simple'; @@ -74,7 +74,6 @@ requires 'HTML::Entities'; requires 'HTML::TokeParser::Simple'; requires 'HTTP::Request::Common'; requires 'Hash::Merge::Simple'; -requires 'Hijk' => '0.27'; requires 'IO::All'; requires 'IO::Interactive'; requires 'IO::Prompt'; @@ -157,9 +156,7 @@ requires 'Ref::Util'; requires 'Regexp::Common'; requires 'Regexp::Common::time'; requires 'Safe', '2.35'; # bug fixes (used by Parse::PMFile) -requires 'Search::Elasticsearch' => '6.00'; -requires 'Search::Elasticsearch::Client::2_0::Direct' => '5.02'; -requires 'Search::Elasticsearch::Cxn::Hijk' => '6.00'; +requires 'Search::Elasticsearch', '== 2.03'; requires 'Starman'; requires 'Throwable::Error'; requires 'Time::Local'; diff --git a/lib/MetaCPAN/Script/CPANTesters.pm b/lib/MetaCPAN/Script/CPANTesters.pm index 498d4f114..f49120dc5 100644 --- a/lib/MetaCPAN/Script/CPANTesters.pm +++ b/lib/MetaCPAN/Script/CPANTesters.pm @@ -10,7 +10,6 @@ use File::stat qw(stat); use IO::Uncompress::Bunzip2 qw(bunzip2); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types qw( Bool File Uri ); -use ElasticSearchX::Model::Document::Types qw(ESBulk); use Moose; with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; @@ -47,7 +46,7 @@ has skip_download => ( has _bulk => ( is => 'ro', - isa => ESBulk, + isa => 'Search::Elasticsearch::Bulk', lazy => 1, default => sub { $_[0]->es->bulk_helper( diff --git a/lib/MetaCPAN/Script/CPANTestersAPI.pm b/lib/MetaCPAN/Script/CPANTestersAPI.pm index b4082521f..a2ee0b094 100644 --- a/lib/MetaCPAN/Script/CPANTestersAPI.pm +++ b/lib/MetaCPAN/Script/CPANTestersAPI.pm @@ -6,7 +6,6 @@ use warnings; use Log::Contextual qw( :log :dlog ); use Cpanel::JSON::XS qw( decode_json ); use MetaCPAN::Types qw( Uri ); -use ElasticSearchX::Model::Document::Types qw(ESBulk); use Moose; with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; @@ -29,7 +28,7 @@ sub _build_url { has _bulk => ( is => 'ro', - isa => ESBulk, + isa => 'Search::Elasticsearch::Bulk', lazy => 1, default => sub { $_[0]->es->bulk_helper( diff --git a/lib/MetaCPAN/Server/Model/CPAN.pm b/lib/MetaCPAN/Server/Model/CPAN.pm index 0e8196fa6..ae547e23f 100644 --- a/lib/MetaCPAN/Server/Model/CPAN.pm +++ b/lib/MetaCPAN/Server/Model/CPAN.pm @@ -22,13 +22,13 @@ has index => ( default => 'cpan', ); -has es_config => ( +has servers => ( is => 'ro', default => ':9200', ); sub _build_esx_model { - MetaCPAN::Model->new( es => shift->es_config ); + MetaCPAN::Model->new( es => shift->servers ); } sub type { diff --git a/metacpan_server.conf b/metacpan_server.conf index ca40a56de..5558ea850 100644 --- a/metacpan_server.conf +++ b/metacpan_server.conf @@ -10,27 +10,3 @@ minion_dsn = postgresql:///minion_queue secret_key 8225b1874fdc431cedb1cf7d454a92b8fde3a5e6 - - - - nodes localhost:9200 - client 2_0::Direct - cxn Hijk - - - - - - nodes localhost:9200 - client 2_0::Direct - cxn Hijk - - - - - - nodes localhost:9200 - client 2_0::Direct - cxn Hijk - - diff --git a/metacpan_server_testing.conf b/metacpan_server_testing.conf index 173aca5c8..59e200d54 100644 --- a/metacpan_server_testing.conf +++ b/metacpan_server_testing.conf @@ -2,21 +2,15 @@ cpan var/t/tmp/fakecpan source_base var/t/tmp/source - - nodes __ENV(ES)__ - + servers __ENV(ES)__ - - nodes __ENV(ES)__ - + servers __ENV(ES)__ - - nodes __ENV(ES)__ - + servers __ENV(ES)__ diff --git a/t/lib/MetaCPAN/TestServer.pm b/t/lib/MetaCPAN/TestServer.pm index bcbc8d04d..5c814a017 100644 --- a/t/lib/MetaCPAN/TestServer.pm +++ b/t/lib/MetaCPAN/TestServer.pm @@ -103,12 +103,11 @@ sub _build_es_server { my $self = shift; my $server = Search::Elasticsearch::TestServer->new( - conf => [ 'cluster.name' => 'metacpan-test' ], - es_home => $self->_es_home, - es_port => 9700, - http_port => 9900, - instances => 1, - es_version => '2_0', + conf => [ 'cluster.name' => 'metacpan-test' ], + es_home => $self->_es_home, + es_port => 9700, + http_port => 9900, + instances => 1, ); diag 'Connecting to Elasticsearch on ' . $self->_es_home; @@ -140,8 +139,7 @@ sub _build_es_client { my $es = Search::Elasticsearch->new( nodes => $self->_es_home, - ( $ENV{ES_TRACE} ? ( trace_to => [ 'File', 'es.log' ] ) : () ), - client => '2_0::Direct', + ( $ENV{ES_TRACE} ? ( trace_to => [ 'File', 'es.log' ] ) : () ) ); ok( $es, 'got ElasticSearch object' ); From ca8e8f3ac9c7f4ce907f3b86794a6912df69816b Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Thu, 28 Jun 2018 00:33:50 +0200 Subject: [PATCH 034/892] Revert "update cpanfile.snapshot" This reverts commit 6fb73c60df4c96c5e4e77645a3df73464994c6f0. --- cpanfile.snapshot | 208 ++++++++++++++++++++-------------------------- 1 file changed, 89 insertions(+), 119 deletions(-) diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 2794c565a..db1763ca7 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -2459,32 +2459,32 @@ DISTRIBUTIONS Canary::Stability 0 ExtUtils::MakeMaker 6.52 common::sense 0 - ElasticSearchX-Model-2.0.0 - pathname: O/OA/OALDERS/ElasticSearchX-Model-2.0.0.tar.gz - provides: - ElasticSearchX::Model v2.0.0 - ElasticSearchX::Model::Bulk v2.0.0 - ElasticSearchX::Model::Document v2.0.0 - ElasticSearchX::Model::Document::EmbeddedRole v2.0.0 - ElasticSearchX::Model::Document::Mapping v2.0.0 - ElasticSearchX::Model::Document::Role v2.0.0 - ElasticSearchX::Model::Document::Set v2.0.0 - ElasticSearchX::Model::Document::Trait::Attribute v2.0.0 - ElasticSearchX::Model::Document::Trait::Class v2.0.0 - ElasticSearchX::Model::Document::Trait::Class::ID v2.0.0 - ElasticSearchX::Model::Document::Trait::Class::Timestamp v2.0.0 - ElasticSearchX::Model::Document::Trait::Class::Version v2.0.0 - ElasticSearchX::Model::Document::Trait::Field::ID v2.0.0 - ElasticSearchX::Model::Document::Trait::Field::TTL v2.0.0 - ElasticSearchX::Model::Document::Trait::Field::Timestamp v2.0.0 - ElasticSearchX::Model::Document::Trait::Field::Version v2.0.0 - ElasticSearchX::Model::Document::Types v2.0.0 - ElasticSearchX::Model::Index v2.0.0 - ElasticSearchX::Model::Role v2.0.0 - ElasticSearchX::Model::Scroll v2.0.0 - ElasticSearchX::Model::Trait::Class v2.0.0 - ElasticSearchX::Model::Tutorial v2.0.0 - ElasticSearchX::Model::Util v2.0.0 + ElasticSearchX-Model-1.0.2 + pathname: O/OA/OALDERS/ElasticSearchX-Model-1.0.2.tar.gz + provides: + ElasticSearchX::Model v1.0.2 + ElasticSearchX::Model::Bulk v1.0.2 + ElasticSearchX::Model::Document v1.0.2 + ElasticSearchX::Model::Document::EmbeddedRole v1.0.2 + ElasticSearchX::Model::Document::Mapping v1.0.2 + ElasticSearchX::Model::Document::Role v1.0.2 + ElasticSearchX::Model::Document::Set v1.0.2 + ElasticSearchX::Model::Document::Trait::Attribute v1.0.2 + ElasticSearchX::Model::Document::Trait::Class v1.0.2 + ElasticSearchX::Model::Document::Trait::Class::ID v1.0.2 + ElasticSearchX::Model::Document::Trait::Class::Timestamp v1.0.2 + ElasticSearchX::Model::Document::Trait::Class::Version v1.0.2 + ElasticSearchX::Model::Document::Trait::Field::ID v1.0.2 + ElasticSearchX::Model::Document::Trait::Field::TTL v1.0.2 + ElasticSearchX::Model::Document::Trait::Field::Timestamp v1.0.2 + ElasticSearchX::Model::Document::Trait::Field::Version v1.0.2 + ElasticSearchX::Model::Document::Types v1.0.2 + ElasticSearchX::Model::Index v1.0.2 + ElasticSearchX::Model::Role v1.0.2 + ElasticSearchX::Model::Scroll v1.0.2 + ElasticSearchX::Model::Trait::Class v1.0.2 + ElasticSearchX::Model::Tutorial v1.0.2 + ElasticSearchX::Model::Util v1.0.2 requirements: Carp 0 Class::Load 0 @@ -2493,10 +2493,10 @@ DISTRIBUTIONS DateTime::Format::ISO8601 0 Digest::SHA1 0 Eval::Closure 0 - ExtUtils::MakeMaker 0 JSON::MaybeXS 0 List::MoreUtils 0 List::Util 0 + Module::Build 0.3601 Module::Find 0 Moose 2.02 Moose::Exporter 0 @@ -2513,8 +2513,9 @@ DISTRIBUTIONS MooseX::Types::Structured 0 Scalar::Util 0 Search::Elasticsearch 2.02 + Search::Elasticsearch::Bulk 0 + Search::Elasticsearch::Scroll 0 Sub::Exporter 0 - perl 5.006 strict 0 version 0 warnings 0 @@ -3602,14 +3603,6 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.008001 - Hijk-0.27 - pathname: G/GU/GUGOD/Hijk-0.27.tar.gz - provides: - Hijk 0.27 - requirements: - CPAN::Meta 0 - ExtUtils::MakeMaker 6.36 - Time::HiRes 0 Hook-LexWrap-0.26 pathname: E/ET/ETHER/Hook-LexWrap-0.26.tar.gz provides: @@ -7554,54 +7547,65 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Test::More 0 perl 5.006001 - Search-Elasticsearch-6.00 - pathname: D/DR/DRTECH/Search-Elasticsearch-6.00.tar.gz - provides: - Search::Elasticsearch 6.00 - Search::Elasticsearch::Client::6_0 6.00 - Search::Elasticsearch::Client::6_0::Bulk 6.00 - Search::Elasticsearch::Client::6_0::Direct 6.00 - Search::Elasticsearch::Client::6_0::Direct::Cat 6.00 - Search::Elasticsearch::Client::6_0::Direct::Cluster 6.00 - Search::Elasticsearch::Client::6_0::Direct::Indices 6.00 - Search::Elasticsearch::Client::6_0::Direct::Ingest 6.00 - Search::Elasticsearch::Client::6_0::Direct::Nodes 6.00 - Search::Elasticsearch::Client::6_0::Direct::Snapshot 6.00 - Search::Elasticsearch::Client::6_0::Direct::Tasks 6.00 - Search::Elasticsearch::Client::6_0::Role::API 6.00 - Search::Elasticsearch::Client::6_0::Role::Bulk 6.00 - Search::Elasticsearch::Client::6_0::Role::Scroll 6.00 - Search::Elasticsearch::Client::6_0::Scroll 6.00 - Search::Elasticsearch::Client::6_0::TestServer 6.00 - Search::Elasticsearch::Cxn::Factory 6.00 - Search::Elasticsearch::Cxn::HTTPTiny 6.00 - Search::Elasticsearch::Cxn::Hijk 6.00 - Search::Elasticsearch::Cxn::LWP 6.00 - Search::Elasticsearch::CxnPool::Sniff 6.00 - Search::Elasticsearch::CxnPool::Static 6.00 - Search::Elasticsearch::CxnPool::Static::NoPing 6.00 - Search::Elasticsearch::Error 6.00 - Search::Elasticsearch::Logger::LogAny 6.00 - Search::Elasticsearch::Role::API 6.00 - Search::Elasticsearch::Role::Client 6.00 - Search::Elasticsearch::Role::Client::Direct 6.00 - Search::Elasticsearch::Role::Cxn 6.00 - Search::Elasticsearch::Role::CxnPool 6.00 - Search::Elasticsearch::Role::CxnPool::Sniff 6.00 - Search::Elasticsearch::Role::CxnPool::Static 6.00 - Search::Elasticsearch::Role::CxnPool::Static::NoPing 6.00 - Search::Elasticsearch::Role::Is_Sync 6.00 - Search::Elasticsearch::Role::Logger 6.00 - Search::Elasticsearch::Role::Serializer 6.00 - Search::Elasticsearch::Role::Serializer::JSON 6.00 - Search::Elasticsearch::Role::Transport 6.00 - Search::Elasticsearch::Serializer::JSON 6.00 - Search::Elasticsearch::Serializer::JSON::Cpanel 6.00 - Search::Elasticsearch::Serializer::JSON::PP 6.00 - Search::Elasticsearch::Serializer::JSON::XS 6.00 - Search::Elasticsearch::TestServer 6.00 - Search::Elasticsearch::Transport 6.00 - Search::Elasticsearch::Util 6.00 + Search-Elasticsearch-2.03 + pathname: D/DR/DRTECH/Search-Elasticsearch-2.03.tar.gz + provides: + Search::Elasticsearch 2.03 + Search::Elasticsearch::Bulk 2.03 + Search::Elasticsearch::Client::0_90::Direct 2.03 + Search::Elasticsearch::Client::0_90::Direct::Cluster 2.03 + Search::Elasticsearch::Client::0_90::Direct::Indices 2.03 + Search::Elasticsearch::Client::1_0::Direct 2.03 + Search::Elasticsearch::Client::1_0::Direct::Cat 2.03 + Search::Elasticsearch::Client::1_0::Direct::Cluster 2.03 + Search::Elasticsearch::Client::1_0::Direct::Indices 2.03 + Search::Elasticsearch::Client::1_0::Direct::Nodes 2.03 + Search::Elasticsearch::Client::1_0::Direct::Snapshot 2.03 + Search::Elasticsearch::Client::2_0::Direct 2.03 + Search::Elasticsearch::Client::2_0::Direct::Cat 2.03 + Search::Elasticsearch::Client::2_0::Direct::Cluster 2.03 + Search::Elasticsearch::Client::2_0::Direct::Indices 2.03 + Search::Elasticsearch::Client::2_0::Direct::Nodes 2.03 + Search::Elasticsearch::Client::2_0::Direct::Snapshot 2.03 + Search::Elasticsearch::Client::2_0::Direct::Tasks 2.03 + Search::Elasticsearch::Cxn::Factory 2.03 + Search::Elasticsearch::Cxn::HTTPTiny 2.03 + Search::Elasticsearch::Cxn::Hijk 2.03 + Search::Elasticsearch::Cxn::LWP 2.03 + Search::Elasticsearch::CxnPool::Sniff 2.03 + Search::Elasticsearch::CxnPool::Static 2.03 + Search::Elasticsearch::CxnPool::Static::NoPing 2.03 + Search::Elasticsearch::Error 2.03 + Search::Elasticsearch::Logger::LogAny 2.03 + Search::Elasticsearch::Role::API::0_90 2.03 + Search::Elasticsearch::Role::API::1_0 2.03 + Search::Elasticsearch::Role::API::2_0 2.03 + Search::Elasticsearch::Role::Bulk 2.03 + Search::Elasticsearch::Role::Client 2.03 + Search::Elasticsearch::Role::Client::Direct 2.03 + Search::Elasticsearch::Role::Client::Direct::Main 2.03 + Search::Elasticsearch::Role::Cxn 2.03 + Search::Elasticsearch::Role::Cxn::HTTP 2.03 + Search::Elasticsearch::Role::CxnPool 2.03 + Search::Elasticsearch::Role::CxnPool::Sniff 2.03 + Search::Elasticsearch::Role::CxnPool::Static 2.03 + Search::Elasticsearch::Role::CxnPool::Static::NoPing 2.03 + Search::Elasticsearch::Role::Is_Sync 2.03 + Search::Elasticsearch::Role::Logger 2.03 + Search::Elasticsearch::Role::Scroll 2.03 + Search::Elasticsearch::Role::Serializer 2.03 + Search::Elasticsearch::Role::Serializer::JSON 2.03 + Search::Elasticsearch::Role::Transport 2.03 + Search::Elasticsearch::Scroll 2.03 + Search::Elasticsearch::Serializer::JSON 2.03 + Search::Elasticsearch::Serializer::JSON::Cpanel 2.03 + Search::Elasticsearch::Serializer::JSON::PP 2.03 + Search::Elasticsearch::Serializer::JSON::XS 2.03 + Search::Elasticsearch::TestServer 2.03 + Search::Elasticsearch::Transport 2.03 + Search::Elasticsearch::Util 2.03 + Search::Elasticsearch::Util::API::Path 2.03 + Search::Elasticsearch::Util::API::QS 2.03 requirements: Any::URI::Escape 0 Data::Dumper 0 @@ -7612,11 +7616,8 @@ DISTRIBUTIONS HTTP::Headers 0 HTTP::Request 0 HTTP::Tiny 0.043 - IO::Compress::Deflate 0 - IO::Compress::Gzip 0 IO::Select 0 IO::Socket 0 - IO::Uncompress::Gunzip 0 IO::Uncompress::Inflate 0 JSON::MaybeXS 1.002002 JSON::PP 0 @@ -7626,7 +7627,7 @@ DISTRIBUTIONS Log::Any::Adapter 0 MIME::Base64 0 Module::Runtime 0 - Moo 2.001000 + Moo 1.003 Moo::Role 0 POSIX 0 Package::Stash 0.34 @@ -7639,37 +7640,6 @@ DISTRIBUTIONS overload 0 strict 0 warnings 0 - Search-Elasticsearch-Client-2_0-5.02 - pathname: D/DR/DRTECH/Search-Elasticsearch-Client-2_0-5.02.tar.gz - provides: - Search::Elasticsearch::Client::2_0 5.02 - Search::Elasticsearch::Client::2_0::Bulk 5.02 - Search::Elasticsearch::Client::2_0::Direct 5.02 - Search::Elasticsearch::Client::2_0::Direct::Cat 5.02 - Search::Elasticsearch::Client::2_0::Direct::Cluster 5.02 - Search::Elasticsearch::Client::2_0::Direct::Indices 5.02 - Search::Elasticsearch::Client::2_0::Direct::Nodes 5.02 - Search::Elasticsearch::Client::2_0::Direct::Snapshot 5.02 - Search::Elasticsearch::Client::2_0::Direct::Tasks 5.02 - Search::Elasticsearch::Client::2_0::Role::API 5.02 - Search::Elasticsearch::Client::2_0::Role::Bulk 5.02 - Search::Elasticsearch::Client::2_0::Role::Scroll 5.02 - Search::Elasticsearch::Client::2_0::Scroll 5.02 - Search::Elasticsearch::Client::2_0::TestServer 5.02 - requirements: - Devel::GlobalDestruction 0 - ExtUtils::MakeMaker 0 - Moo 0 - Moo::Role 0 - Search::Elasticsearch 5.02 - Search::Elasticsearch::Role::API 0 - Search::Elasticsearch::Role::Client::Direct 0 - Search::Elasticsearch::Role::Is_Sync 0 - Search::Elasticsearch::Util 0 - Try::Tiny 0 - namespace::clean 0 - strict 0 - warnings 0 Server-Starter-0.33 pathname: K/KA/KAZUHO/Server-Starter-0.33.tar.gz provides: From f81147127a8727b9b4855641a6390390a241e091 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 30 Jun 2018 22:51:21 +0100 Subject: [PATCH 035/892] Increase alarm for modules extraction from files In some cases (see #812) release have many files and no provides this timeout is simply not enough in those cases and it's timing out is silenced in try/catch which causes a partial indexing with missing 'modules' section which causes no 'latest' bit marking which makes the release not discoverable. --- lib/MetaCPAN/Model/Release.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 93e5064b4..9a0245049 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -540,7 +540,7 @@ sub _modules_from_files { log_error {'Call to Module::Metadata timed out '}; die; }; - alarm(5); + alarm(50); my $info; { local $SIG{__WARN__} = sub { }; From 87b9dc757cd0d32b1f23fa14c63968fb9d8b7854 Mon Sep 17 00:00:00 2001 From: "Philippe Bruhat (BooK)" Date: Thu, 2 Aug 2018 17:38:26 +0200 Subject: [PATCH 036/892] Update the DSN to the UDD The address and credentials have changed. See https://udd-mirror.debian.net/ for details. --- lib/MetaCPAN/Script/Role/External/Debian.pm | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/lib/MetaCPAN/Script/Role/External/Debian.pm b/lib/MetaCPAN/Script/Role/External/Debian.pm index edc3235cc..b74382d70 100644 --- a/lib/MetaCPAN/Script/Role/External/Debian.pm +++ b/lib/MetaCPAN/Script/Role/External/Debian.pm @@ -42,8 +42,8 @@ sub run_debian { # connect to the database my $dbh = DBI->connect( - "dbi:Pg:host=public-udd-mirror.xvm.mit.edu;dbname=udd", - 'public-udd-mirror', 'public-udd-mirror' ); + "dbi:Pg:host=udd-mirror.debian.net;dbname=udd", + 'udd-mirror', 'udd-mirror' ); # special cases my %skip = ( 'libbssolv-perl' => 1 ); From dfa9a53444cd65bfa60fbccbed42928fd96dd1dd Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Mon, 6 Aug 2018 16:00:14 +0100 Subject: [PATCH 037/892] tidy fix: Role::External::Debian --- lib/MetaCPAN/Script/Role/External/Debian.pm | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Script/Role/External/Debian.pm b/lib/MetaCPAN/Script/Role/External/Debian.pm index b74382d70..7761fec77 100644 --- a/lib/MetaCPAN/Script/Role/External/Debian.pm +++ b/lib/MetaCPAN/Script/Role/External/Debian.pm @@ -40,9 +40,7 @@ sub run_debian { my $ret = {}; # connect to the database - my $dbh - = DBI->connect( - "dbi:Pg:host=udd-mirror.debian.net;dbname=udd", + my $dbh = DBI->connect( "dbi:Pg:host=udd-mirror.debian.net;dbname=udd", 'udd-mirror', 'udd-mirror' ); # special cases From 697d619062c7cc4129c8fae938f406b6ea5d0c1f Mon Sep 17 00:00:00 2001 From: "Jonas B. Nielsen" Date: Mon, 6 Aug 2018 23:08:49 +0200 Subject: [PATCH 038/892] Referenced URL unresponsive The URL does not respond on https, but http. --- docs/API-docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API-docs.md b/docs/API-docs.md index c74a7c203..b2096a9be 100644 --- a/docs/API-docs.md +++ b/docs/API-docs.md @@ -8,7 +8,7 @@ _All of these URLs can be tested using the [MetaCPAN Explorer](https://explorer. To learn more about the ElasticSearch query DSL (Domain-Specific Language) check out Clinton Gormley's [Terms of Endearment - ES Query DSL Explained] (https://www.slideshare.net/clintongormley/terms-of-endearment-the-elasticsearch-query-dsl-explained) slides. -The query syntax is explained on ElasticSearch's [reference page](https://www.elasticsearch.org/guide/reference/query-dsl/). You can also check out this getting started tutorial about Elasticsearch [reference page](https://joelabrahamsson.com/elasticsearch-101/). +The query syntax is explained on ElasticSearch's [reference page](https://www.elasticsearch.org/guide/reference/query-dsl/). You can also check out this getting started tutorial about Elasticsearch [reference page](http://joelabrahamsson.com/elasticsearch-101/). ## Being polite From 6f739aa492ff43c2d7924b7d8085ca2455d18552 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Fri, 19 Oct 2018 09:50:14 -0400 Subject: [PATCH 039/892] Add definition for Mac::SystemDirectory This module is required by File::HomeDir but only when installed on a Mac. It's an optional prerequisite. Having the definition in the cpanfile.snapshot allows carton install --deployment to install Mac::SystemDirectory if required, and has no impact if it is not. --- cpanfile.snapshot | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/cpanfile.snapshot b/cpanfile.snapshot index db1763ca7..9dff276ef 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4281,6 +4281,17 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.006 + Mac-SystemDirectory-0.10 + pathname: E/ET/ETHER/Mac-SystemDirectory-0.10.tar.gz + provides: + Mac::SystemDirectory 0.10 + requirements: + Exporter 0 + ExtUtils::MakeMaker 0 + XSLoader 0 + perl 5.006 + strict 0 + warnings 0 MailTools-2.19 pathname: M/MA/MARKOV/MailTools-2.19.tar.gz provides: From c1ffa7fb11da2a7a0369780c99e5b12c002b87b4 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Mon, 22 Oct 2018 07:49:22 +0100 Subject: [PATCH 040/892] Script::Release - allow forcing authorized flag In some cases we have to deal with incorrect data based on reindexing after permissions have changed. In those cases we may want to reindex (manually) while forcing the check of authorized to 'true' knowing this was the case at the time of the original release. See GH#2117 (https://github.com/metacpan/metacpan-web/issues/2117) --- lib/MetaCPAN/Script/Release.pm | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index 4a3c2b9e5..511759da9 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -42,6 +42,13 @@ has skip => ( documentation => 'skip already indexed modules (0)', ); +has force_authorized => ( + is => 'ro', + isa => Bool, + default => 0, + documentation => 'force authorized when indexing (0)', +); + has status => ( is => 'ro', isa => Str, @@ -255,12 +262,14 @@ sub import_archive { # NOTE: "The method returns a list of unauthorized, but indexed modules." push( @release_unauthorized, $file->set_authorized($perms) ) - if ( keys %$perms ); + if ( keys %$perms and !$self->force_authorized ); my $file_x_deprecated = 0; for ( @{ $file->module } ) { - push( @provides, $_->name ) if $_->indexed && $_->authorized; + push( @provides, $_->name ) + if $_->indexed + && ( $_->authorized || $self->force_authorized ); $file_x_deprecated = 1 if $meta->{provides}{ $_->name }{x_deprecated}; } From be2c1fd4b1360119ec6cb98c9021ba9b74177590 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 8 Nov 2018 13:28:31 -0600 Subject: [PATCH 041/892] Add Travis DarkPAN caching So that tests don't fail if a tarball cannot be fetched. --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3fa01ded4..b79635126 100644 --- a/.travis.yml +++ b/.travis.yml @@ -91,6 +91,7 @@ cache: directories: - local - ~/perl5 + - t/var/darkpan addons: artifacts: From b402f371e048d30686346415e1a4fcb7c8932d72 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Thu, 8 Nov 2018 21:45:32 +0000 Subject: [PATCH 042/892] Added purge script --- lib/MetaCPAN/Script/Purge.pm | 436 +++++++++++++++++++++++++++++++++++ 1 file changed, 436 insertions(+) create mode 100644 lib/MetaCPAN/Script/Purge.pm diff --git a/lib/MetaCPAN/Script/Purge.pm b/lib/MetaCPAN/Script/Purge.pm new file mode 100644 index 000000000..7c3d2f43e --- /dev/null +++ b/lib/MetaCPAN/Script/Purge.pm @@ -0,0 +1,436 @@ +package MetaCPAN::Script::Purge; + +use Moose; + +use Log::Contextual qw( :log ); +use MetaCPAN::Types qw( Bool Str HashRef ); + +with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; + +has author => ( + is => 'ro', + isa => Str, + required => 1, +); + +has release => ( + is => 'ro', + isa => Str, + required => 0, +); + +has force => ( + is => 'ro', + isa => Bool, + default => 0, +); + +has bulk => ( + is => 'ro', + isa => HashRef, + lazy => 1, + builder => '_build_bulk', +); + +sub _build_bulk { + my $self = shift; + my $index = $self->index->name; + return +{ + author => $self->es->bulk_helper( index => $index, type => 'author' ), + contributor => $self->es->bulk_helper( + index => 'contributor', + type => 'contributor' + ), + favorite => + $self->es->bulk_helper( index => $index, type => 'favorite' ), + file => $self->es->bulk_helper( index => $index, type => 'file' ), + permission => + $self->es->bulk_helper( index => $index, type => 'permission' ), + rating => $self->es->bulk_helper( index => $index, type => 'rating' ), + release => + $self->es->bulk_helper( index => $index, type => 'release' ), + }; +} + +sub _get_scroller_release { + my ( $self, $query ) = @_; + return $self->es->scroll_helper( + size => 500, + scroll => '10m', + index => $self->index->name, + type => 'release', + body => { query => $query }, + fields => [qw( name )], + ); +} + +sub _get_scroller_rating { + my ( $self, $query ) = @_; + return $self->es->scroll_helper( + size => 500, + scroll => '10m', + index => $self->index->name, + type => 'rating', + body => { query => $query }, + fields => [], + ); +} + +sub _get_scroller_file { + my ( $self, $query ) = @_; + return $self->es->scroll_helper( + size => 500, + scroll => '10m', + index => $self->index->name, + type => 'file', + body => { query => $query }, + fields => [qw( name )], + ); +} + +sub _get_scroller_favorite { + my ( $self, $query ) = @_; + return $self->es->scroll_helper( + size => 500, + scroll => '10m', + index => $self->index->name, + type => 'favorite', + body => { query => $query }, + fields => [], + ); +} + +sub _get_scroller_contributor { + my ( $self, $query ) = @_; + return $self->es->scroll_helper( + size => 500, + scroll => '10m', + index => 'contributor', + type => 'contributor', + body => { query => $query }, + fields => [qw( release_name )], + ); +} + +sub run { + my $self = shift; + + if ( $self->author ) { + if ( !$self->force ) { + if ( $self->release ) { + $self->are_you_sure( 'Release ' + . $self->release + . ' by author ' + . $self->author + . ' will be removed from the index !!!' ); + } + else { + $self->are_you_sure( 'Author ' + . $self->author + . ' + all their releases will be removed from the index !!!' + ); + } + } + $self->purge_author_releases; + $self->purge_favorite; + $self->purge_author; + $self->purge_contributor; + $self->purge_rating; + } + + $self->index->refresh; +} + +sub purge_author_releases { + my $self = shift; + + if ( $self->release ) { + $self->purge_single_release; + $self->purge_files( $self->release ); + } + else { + $self->purge_multiple_releases; + } +} + +sub purge_single_release { + my $self = shift; + log_info { + 'Looking for release ' + . $self->release + . ' by author ' + . $self->author + }; + + my $query = { + bool => { + must => [ + { term => { author => $self->author } }, + { term => { name => $self->release } } + ] + } + }; + + my $scroll = $self->_get_scroller_release($query); + my @remove; + + while ( my $r = $scroll->next ) { + log_debug { 'Removing release ' . $r->{fields}{name}[0] }; + push @remove, $r->{_id}; + } + + if (@remove) { + $self->bulk->{release}->delete_ids(@remove); + $self->bulk->{release}->flush; + } + + log_info { 'Finished purging release ' . $self->release }; +} + +sub purge_multiple_releases { + my $self = shift; + log_info { 'Looking all up author ' . $self->author . ' releases' }; + + my $query = { term => { author => $self->author } }; + + my $scroll = $self->_get_scroller_release($query); + my @remove_ids; + my @remove_release_files; + + while ( my $r = $scroll->next ) { + log_debug { 'Removing release ' . $r->{fields}{name}[0] }; + push @remove_ids, $r->{_id}; + push @remove_release_files, $r->{fields}{name}[0]; + } + + if (@remove_ids) { + $self->bulk->{release}->delete_ids(@remove_ids); + $self->bulk->{release}->flush; + } + + for my $release (@remove_release_files) { + $self->purge_files($release); + } + + log_info { 'Finished purging releases for author ' . $self->author }; +} + +sub purge_files { + my ( $self, $release ) = @_; + log_info { + 'Looking for files of release ' + . $release + . ' by author ' + . $self->author + }; + + my $query = { + bool => { + must => [ + { term => { author => $self->author } }, + { term => { release => $release } } + ] + } + }; + + my $scroll = $self->_get_scroller_file($query); + my @remove; + + while ( my $f = $scroll->next ) { + log_debug { + 'Removing file ' + . $f->{fields}{name}[0] + . ' of release ' + . $release + }; + push @remove, $f->{_id}; + } + + if (@remove) { + $self->bulk->{file}->delete_ids(@remove); + $self->bulk->{file}->flush; + } + + log_info { 'Finished purging files for release ' . $release }; +} + +sub purge_favorite { + my ( $self, $release ) = @_; + + if ( $self->release ) { + log_info { + 'Looking for favorites of release ' + . $self->release + . ' by author ' + . $self->author + }; + $self->_purge_favorite( { term => { release => $self->release } } ); + log_info { + 'Finished purging favorites for release ' . $self->release + }; + } + else { + log_info { 'Looking for favorites author ' . $self->author }; + $self->_purge_favorite( { term => { author => $self->author } } ); + log_info { 'Finished purging favorites for author ' . $self->author }; + } +} + +sub _purge_favorite { + my ( $self, $query ) = @_; + + my $scroll = $self->_get_scroller_favorite($query); + my @remove; + + while ( my $f = $scroll->next ) { + push @remove, $f->{_id}; + } + + if (@remove) { + $self->bulk->{favorite}->delete_ids(@remove); + $self->bulk->{favorite}->flush; + } +} + +sub purge_author { + my $self = shift; + log_info { 'Purging author ' . $self->author }; + + $self->bulk->{author}->delete_ids( $self->author ); + $self->bulk->{author}->flush; + + log_info { 'Finished purging author ' . $self->author }; +} + +sub purge_contributor { + my $self = shift; + log_info { 'Looking all up author ' . $self->author . ' contributions' }; + + my @remove; + + my $query_release_author + = { term => { release_author => $self->author } }; + + my $scroll_release_author + = $self->_get_scroller_contributor($query_release_author); + + while ( my $r = $scroll_release_author->next ) { + log_debug { + 'Removing contributions to releases by author ' . $self->author + }; + push @remove, $r->{_id}; + } + + my $query_pauseid = { term => { pauseid => $self->author } }; + + my $scroll_pauseid = $self->_get_scroller_contributor($query_pauseid); + + while ( my $c = $scroll_pauseid->next ) { + log_debug { 'Removing contributions of author ' . $self->author }; + push @remove, $c->{_id}; + } + + if (@remove) { + $self->bulk->{contributor}->delete_ids(@remove); + $self->bulk->{contributor}->flush; + } + + log_info { + 'Finished purging contribution entries related to ' . $self->author + }; +} + +sub purge_rating { + my $self = shift; + + if ( $self->release ) { + $self->purge_rating_release; + } + else { + $self->purge_rating_author; + } +} + +sub purge_rating_release { + my $self = shift; + log_info { + 'Looking all up ratings for release ' + . $self->release + . ' author ' + . $self->author + }; + + my @remove; + + my $query = { + bool => { + must => [ + { term => { author => $self->author } }, + { term => { release => $self->release } } + ] + } + }; + + my $scroll_rating = $self->_get_scroller_rating($query); + + while ( my $r = $scroll_rating->next ) { + log_debug { + 'Removing ratings for release ' + . $self->release + . ' by author ' + . $self->author + }; + push @remove, $r->{_id}; + } + + if (@remove) { + $self->bulk->{rating}->delete_ids(@remove); + $self->bulk->{rating}->flush; + } + + log_info { + 'Finished purging rating entries for release ' + . $self->release + . ' by author ' + . $self->author + }; +} + +sub purge_rating_author { + my $self = shift; + log_info { 'Looking all up ratings for author ' . $self->author }; + + my @remove; + + my $query = { term => { author => $self->author } }; + + my $scroll_rating = $self->_get_scroller_rating($query); + + while ( my $r = $scroll_rating->next ) { + log_debug { 'Removing ratings related to author ' . $self->author }; + push @remove, $r->{_id}; + } + + if (@remove) { + $self->bulk->{rating}->delete_ids(@remove); + $self->bulk->{rating}->flush; + } + + log_info { + 'Finished purging rating entries related to author ' . $self->author + }; +} + +__PACKAGE__->meta->make_immutable; +1; + +=pod + +=head1 SYNOPSIS + +Purge releases from the index, by author or name + + $ bin/metacpan purge --author X + $ bin/metacpan purge --release Y + +=cut From a9e10c62eb412bd10346d324bcf518f3c9000d3e Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Fri, 9 Nov 2018 16:29:19 +0000 Subject: [PATCH 043/892] Script::Purge - quarantine purged archives --- lib/MetaCPAN/Role/Script.pm | 16 +++++++ lib/MetaCPAN/Script/Purge.pm | 86 ++++++++++++++++-------------------- 2 files changed, 53 insertions(+), 49 deletions(-) diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm index b8fc36c56..9657573dd 100644 --- a/lib/MetaCPAN/Role/Script.pm +++ b/lib/MetaCPAN/Role/Script.pm @@ -12,6 +12,7 @@ use MetaCPAN::Queue (); use Term::ANSIColor qw( colored ); use IO::Interactive qw( is_interactive ); use IO::Prompt; +use File::Path (); use Carp (); @@ -86,6 +87,13 @@ has home => ( default => sub { checkout_root() }, ); +has quarantine => ( + is => 'ro', + isa => Str, + lazy => 1, + builder => '_build_quarantine', +); + has _minion => ( is => 'ro', isa => 'Minion', @@ -167,6 +175,14 @@ sub _build_cpan { } +sub _build_quarantine { + my $path = "$ENV{HOME}/QUARANTINE"; + if ( !-d $path ) { + File::Path::mkpath($path); + } + return $path; +} + sub remote { shift->es->nodes->info->[0]; } diff --git a/lib/MetaCPAN/Script/Purge.pm b/lib/MetaCPAN/Script/Purge.pm index 7c3d2f43e..59f53606b 100644 --- a/lib/MetaCPAN/Script/Purge.pm +++ b/lib/MetaCPAN/Script/Purge.pm @@ -4,6 +4,7 @@ use Moose; use Log::Contextual qw( :log ); use MetaCPAN::Types qw( Bool Str HashRef ); +use MetaCPAN::Util qw( author_dir ); with 'MooseX::Getopt', 'MetaCPAN::Role::Script'; @@ -60,7 +61,7 @@ sub _get_scroller_release { index => $self->index->name, type => 'release', body => { query => $query }, - fields => [qw( name )], + fields => [qw( archive name )], ); } @@ -72,7 +73,6 @@ sub _get_scroller_rating { index => $self->index->name, type => 'rating', body => { query => $query }, - fields => [], ); } @@ -96,7 +96,6 @@ sub _get_scroller_favorite { index => $self->index->name, type => 'favorite', body => { query => $query }, - fields => [], ); } @@ -145,62 +144,46 @@ sub purge_author_releases { my $self = shift; if ( $self->release ) { - $self->purge_single_release; - $self->purge_files( $self->release ); - } - else { - $self->purge_multiple_releases; - } -} - -sub purge_single_release { - my $self = shift; - log_info { - 'Looking for release ' - . $self->release - . ' by author ' - . $self->author - }; - - my $query = { - bool => { - must => [ - { term => { author => $self->author } }, - { term => { name => $self->release } } - ] - } - }; + log_info { + 'Looking for release ' + . $self->release + . ' by author ' + . $self->author + }; - my $scroll = $self->_get_scroller_release($query); - my @remove; + my $query = { + bool => { + must => [ + { term => { author => $self->author } }, + { term => { name => $self->release } } + ] + } + }; - while ( my $r = $scroll->next ) { - log_debug { 'Removing release ' . $r->{fields}{name}[0] }; - push @remove, $r->{_id}; + $self->_purge_release($query); + log_info { 'Finished purging release ' . $self->release }; } - - if (@remove) { - $self->bulk->{release}->delete_ids(@remove); - $self->bulk->{release}->flush; + else { + log_info { 'Looking all up author ' . $self->author . ' releases' }; + my $query = { term => { author => $self->author } }; + $self->_purge_release($query); + log_info { 'Finished purging releases for author ' . $self->author }; } - - log_info { 'Finished purging release ' . $self->release }; } -sub purge_multiple_releases { - my $self = shift; - log_info { 'Looking all up author ' . $self->author . ' releases' }; - - my $query = { term => { author => $self->author } }; +sub _purge_release { + my ( $self, $query ) = @_; my $scroll = $self->_get_scroller_release($query); my @remove_ids; my @remove_release_files; + my @remove_release_archives; while ( my $r = $scroll->next ) { log_debug { 'Removing release ' . $r->{fields}{name}[0] }; - push @remove_ids, $r->{_id}; - push @remove_release_files, $r->{fields}{name}[0]; + push @remove_ids, $r->{_id}; + push @remove_release_files, $r->{fields}{name}[0]; + push @remove_release_archives, $r->{fields}{archive}[0]; } if (@remove_ids) { @@ -209,13 +192,18 @@ sub purge_multiple_releases { } for my $release (@remove_release_files) { - $self->purge_files($release); + $self->_purge_files($release); } - log_info { 'Finished purging releases for author ' . $self->author }; + # remove the release archive + for my $archive (@remove_release_archives) { + log_info { "Moving archive $archive to " . $self->quarantine }; + $self->cpan->file( 'authors', author_dir( $self->author ), $archive ) + ->move_to( $self->quarantine ); + } } -sub purge_files { +sub _purge_files { my ( $self, $release ) = @_; log_info { 'Looking for files of release ' From 9688dd990365de482ab20c60392cd3428c0d83ec Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Fri, 9 Nov 2018 18:38:13 +0000 Subject: [PATCH 044/892] Removed unused endpoint /search/simple --- lib/MetaCPAN/Model/Search.pm | 7 ------- lib/MetaCPAN/Server/Controller/Search/Web.pm | 15 +-------------- lib/MetaCPAN/Server/Model/Search.pm | 2 +- 3 files changed, 2 insertions(+), 22 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index bc357e31e..144c56660 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -35,13 +35,6 @@ sub _not_rogue { return { not => { filter => { or => \@rogue_dists } } }; } -sub search_simple { - my ( $self, $search_term ) = @_; - my $es_query = $self->build_query($search_term); - my $es_results = $self->run_query( file => $es_query ); - return $es_results; -} - sub search_for_first_result { my ( $self, $search_term ) = @_; my $es_query = $self->build_query($search_term); diff --git a/lib/MetaCPAN/Server/Controller/Search/Web.pm b/lib/MetaCPAN/Server/Controller/Search/Web.pm index 8c85615a9..d309b9810 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Web.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Web.pm @@ -13,20 +13,7 @@ with 'MetaCPAN::Server::Role::JSONP'; sub get { } sub all { } -# The simple search avoids most of the input and output aggregation and munging and is therefore easier to reason about for say search optimization. - -sub simple : Chained('/search/index') : PathPart('simple') : Args(0) { - my ( $self, $c ) = @_; - my $args = $c->req->params; - - my $model = $c->model('Search'); - my $results = $model->search_simple( $args->{q} ); - - $c->stash($results); -} - -# returns the contents of the first result of a query similar to -# the one done by 'search_simple' +# returns the contents of the first result of a query sub first : Chained('/search/index') : PathPart('first') : Args(0) { my ( $self, $c ) = @_; my $args = $c->req->params; diff --git a/lib/MetaCPAN/Server/Model/Search.pm b/lib/MetaCPAN/Server/Model/Search.pm index 44d8354eb..bba8f2163 100644 --- a/lib/MetaCPAN/Server/Model/Search.pm +++ b/lib/MetaCPAN/Server/Model/Search.pm @@ -12,7 +12,7 @@ has search => ( is => 'ro', isa => 'MetaCPAN::Model::Search', lazy => 1, - handles => [qw( search_for_first_result search_simple search_web )], + handles => [qw( search_for_first_result search_web )], default => sub { my $self = shift; return MetaCPAN::Model::Search->new( From 11be94d29188b9fb612ababa14bb52fed8ac4dae Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Fri, 9 Nov 2018 19:44:44 +0000 Subject: [PATCH 045/892] Script::Purge - fix author purging condition --- lib/MetaCPAN/Script/Purge.pm | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/lib/MetaCPAN/Script/Purge.pm b/lib/MetaCPAN/Script/Purge.pm index 59f53606b..36ee58464 100644 --- a/lib/MetaCPAN/Script/Purge.pm +++ b/lib/MetaCPAN/Script/Purge.pm @@ -132,9 +132,11 @@ sub run { } $self->purge_author_releases; $self->purge_favorite; - $self->purge_author; - $self->purge_contributor; $self->purge_rating; + if ( !$self->release ) { + $self->purge_author; + $self->purge_contributor; + } } $self->index->refresh; From 2b3fbfba2eeec1f36d060829f4fb37286f450789 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 17:00:13 -0600 Subject: [PATCH 046/892] Changes endpoint now recognizes more kinds of Changelog files --- lib/MetaCPAN/Document/File/Set.pm | 14 +++++++++++++- 1 file changed, 13 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index 5f5ef3062..6922b2a51 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -616,7 +616,19 @@ sub find_changes_files { # and store the result as { 'changes_file' => $name } my @candidates = qw( - CHANGES Changes ChangeLog Changelog CHANGELOG NEWS + CHANGELOG + ChangeLog + Changelog + ChangeLog.pm + changelog.pm + ChangeLog.pod + CHANGES + Changes + CHANGES.pm + Changes.pm + CHANGES.pod + Changes.pod + NEWS ); # use $c->model b/c we can't let any filters apply here From 48af4f014fac862d493b93c0395d1ac8d7227a69 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Fri, 9 Nov 2018 23:25:16 +0000 Subject: [PATCH 047/892] Consolidate /search/web & /search/web/v2 /search/web is not used anymore (since June 2018) This is first step for removing v2 after Web starts using /search/web again. --- lib/MetaCPAN/Server/Controller/Search/Web.pm | 9 ++------- 1 file changed, 2 insertions(+), 7 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Search/Web.pm b/lib/MetaCPAN/Server/Controller/Search/Web.pm index d309b9810..3d0b5301c 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Web.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Web.pm @@ -30,13 +30,8 @@ sub web : Chained('/search/index') : PathPart('web') : Args(0) { my ( $self, $c ) = @_; my $args = $c->req->params; - my $model = $c->model('Search'); - my $results - = $model->search_web( @{$args}{qw( q from size collapsed )}, 500 ); - - for my $result ( @{ $results->{results} } ) { - $result = $result->{hits}; - } + my $model = $c->model('Search'); + my $results = $model->search_web( @{$args}{qw( q from size collapsed )} ); $c->stash($results); } From b773381e6a88b4da2ab79f972216aa5e9569bc7f Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 15:35:51 -0600 Subject: [PATCH 048/892] Have Travis cache $HOME/local --- .travis.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.travis.yml b/.travis.yml index 3fa01ded4..6711a3bff 100644 --- a/.travis.yml +++ b/.travis.yml @@ -89,6 +89,7 @@ services: # caching /local should save about 5 minutes in module install time cache: directories: + - $HOME/local - local - ~/perl5 From 67771efd93d27613ca2de5f4a411322d6142d574 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 15:58:39 -0600 Subject: [PATCH 049/892] Don't make Travis build logs too big to view --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 6711a3bff..0beb8870d 100644 --- a/.travis.yml +++ b/.travis.yml @@ -67,7 +67,7 @@ before_install: - cpanm -n Safe@2.35 install: - - AUTHOR_TESTING=0 cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (cat ~/.perl-cpm/build.log; false) + - AUTHOR_TESTING=0 cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (tail -n 500 -f ~/.perl-cpm/build.log; false) before_script: - "perl -i -pe 's/(servers :)9900/localhost:9200/' metacpan_server_testing.conf" From b5e81198f30a4c9b95658b43d86cde78c08f165a Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 17:36:19 -0600 Subject: [PATCH 050/892] Ensure that provides is unique --- lib/MetaCPAN/Script/Release.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index 511759da9..93076dead 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -10,6 +10,7 @@ BEGIN { use CPAN::DistnameInfo (); use File::Find::Rule; use File::stat (); +use List::Util qw( uniq ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Util; use MetaCPAN::Model::Release; @@ -293,7 +294,7 @@ sub import_archive { } } if (@provides) { - $document->_set_provides( [ sort @provides ] ); + $document->_set_provides( [ uniq sort @provides ] ); $document->put; } $bulk->commit; From 61cb0c6dd4734d64696638478585ae02c19db75c Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 17:37:24 -0600 Subject: [PATCH 051/892] Less implicit imports --- lib/MetaCPAN/Script/Release.pm | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index 93076dead..36310d410 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -8,13 +8,13 @@ BEGIN { } use CPAN::DistnameInfo (); -use File::Find::Rule; -use File::stat (); +use File::Find::Rule (); +use File::stat (); use List::Util qw( uniq ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Util; -use MetaCPAN::Model::Release; -use MetaCPAN::Script::Runner; +use MetaCPAN::Model::Release (); +use MetaCPAN::Script::Runner (); use MetaCPAN::Types qw( Bool Dir HashRef Int Str ); use Moose; use PerlIO::gzip; From 88645e401a1674512f61bb5d2c12ffbe0693229e Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 00:45:59 +0000 Subject: [PATCH 052/892] Search::Web - removed web_v2 The code was consolidated with 'sub web' it's no longer in use by Web as well. --- lib/MetaCPAN/Server/Controller/Search/Web.pm | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Search/Web.pm b/lib/MetaCPAN/Server/Controller/Search/Web.pm index 3d0b5301c..ed79ee2f3 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Web.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Web.pm @@ -36,14 +36,4 @@ sub web : Chained('/search/index') : PathPart('web') : Args(0) { $c->stash($results); } -sub web_v2 : Chained('/search/index') : PathPart('web/v2') : Args(0) { - my ( $self, $c ) = @_; - my $args = $c->req->params; - - my $model = $c->model('Search'); - my $results = $model->search_web( @{$args}{qw( q from size collapsed )} ); - - $c->stash($results); -} - 1; From 5c712292f61ec80add493a9467ca47af8b94bea4 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 17:13:10 +0000 Subject: [PATCH 053/892] Search::Web - 404 from API's /search/first endpoint --- lib/MetaCPAN/Server/Controller/Search/Web.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Controller/Search/Web.pm b/lib/MetaCPAN/Server/Controller/Search/Web.pm index 3d0b5301c..dd5db2791 100644 --- a/lib/MetaCPAN/Server/Controller/Search/Web.pm +++ b/lib/MetaCPAN/Server/Controller/Search/Web.pm @@ -21,7 +21,7 @@ sub first : Chained('/search/index') : PathPart('first') : Args(0) { my $model = $c->model('Search'); my $results = $model->search_for_first_result( $args->{q} ); - $c->stash($results) if $results; + $c->stash_or_detach($results); } # The web endpoint is the primary one, this handles the front-end's user-facing search From 6a385c7c5b15f3cd1779d73faa6afd86f84beedd Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 19:20:05 +0000 Subject: [PATCH 054/892] Cover - added URL to endpoint returned data --- lib/MetaCPAN/Query/Cover.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Query/Cover.pm b/lib/MetaCPAN/Query/Cover.pm index 4a7b9a6d2..bd4ad47c1 100644 --- a/lib/MetaCPAN/Query/Cover.pm +++ b/lib/MetaCPAN/Query/Cover.pm @@ -19,7 +19,10 @@ sub find_release_coverage { ); $res->{hits}{total} or return {}; - return $res->{hits}{hits}[0]{_source}; + return +{ + %{ $res->{hits}{hits}[0]{_source} }, + url => "http://cpancover.com/latest/$release/index.html", + }; } __PACKAGE__->meta->make_immutable; From c7d4c377116b5042bd286bda03cb7dc4e10288a7 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 19:21:06 +0000 Subject: [PATCH 055/892] Cover - added test for script+endpoint --- t/script/cover.t | 74 +++++++++++++++++++++++++++++++++++++++++++++ t/var/cover.json | 79 ++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 153 insertions(+) create mode 100644 t/script/cover.t create mode 100644 t/var/cover.json diff --git a/t/script/cover.t b/t/script/cover.t new file mode 100644 index 000000000..0b3a44e6c --- /dev/null +++ b/t/script/cover.t @@ -0,0 +1,74 @@ +use strict; +use warnings; +use lib 't/lib'; + +use Git::Helpers qw( checkout_root ); +use MetaCPAN::Script::Cover (); +use MetaCPAN::Script::Runner (); +use MetaCPAN::Server::Test qw( app GET test_psgi ); +use MetaCPAN::TestHelpers qw( decode_json_ok ); +use Test::More; +use URI (); + +my $config = MetaCPAN::Script::Runner::build_config; + +my $root = checkout_root(); +my $file = URI->new('t/var/cover.json')->abs("file://$root/"); +$config->{'cover_url'} = "$file"; + +my $cover = MetaCPAN::Script::Cover->new_with_options($config); +ok $cover->run, 'runs and returns true'; + +my %expect = ( + 'Devel-GoFaster-0.000' => { + criteria => { + branch => '12.50', + condition => '0.00', + statement => '63.64', + subroutine => '71.43', + total => '46.51', + }, + distribution => 'Devel-GoFaster', + release => 'Devel-GoFaster-0.000', + url => 'http://cpancover.com/latest/Devel-GoFaster-0.000/index.html', + version => '0.000', + }, + 'Try-Tiny-0.27' => { + criteria => { + branch => '78.95', + condition => '46.67', + statement => '95.06', + subroutine => '100.00', + total => '86.58', + }, + distribution => 'Try-Tiny', + release => 'Try-Tiny-0.27', + url => 'http://cpancover.com/latest/Try-Tiny-0.27/index.html', + version => '0.27', + }, +); + +my $test = Plack::Test->create( app() ); + +for my $release ( keys %expect ) { + my $expected = $expect{$release}; + subtest "Check $release" => sub { + my $url = "/cover/$release"; + my $res = $test->request( GET $url ); + diag "GET $url"; + + # TRAVIS 5.18 + is( $res->code, 200, "code 200" ); + is( + $res->header('content-type'), + 'application/json; charset=utf-8', + 'Content-type' + ); + my $json = decode_json_ok($res); + + # TRAVIS 5.18 + is_deeply( $json, $expected, "$release cover summary roundtrip" ); + }; +} + +done_testing(); diff --git a/t/var/cover.json b/t/var/cover.json new file mode 100644 index 000000000..d8dfe901c --- /dev/null +++ b/t/var/cover.json @@ -0,0 +1,79 @@ +{ + "Devel-GoFaster" : { + "0.000" : { + "coverage" : { + "total" : { + "branch" : "12.50", + "condition" : "0.00", + "statement" : "63.64", + "subroutine" : "71.43", + "total" : "46.51" + } + } + }, + "0.001" : { + "coverage" : { + "total" : { + "branch" : "12.50", + "condition" : "0.00", + "statement" : "61.90", + "subroutine" : "71.43", + "total" : "45.24" + } + } + } + }, + "Try-Tiny" : { + "0.22" : { + "coverage" : { + "total" : {} + } + }, + "0.23" : { + "coverage" : { + "total" : {} + } + }, + "0.24" : { + "coverage" : { + "total" : {} + } + }, + "0.27" : { + "coverage" : { + "total" : { + "branch" : "78.95", + "condition" : "46.67", + "pod" : "100.00", + "statement" : "95.06", + "subroutine" : "100.00", + "total" : "86.58" + } + } + }, + "0.28" : { + "coverage" : { + "total" : { + "branch" : "78.95", + "condition" : "46.67", + "pod" : "100.00", + "statement" : "95.06", + "subroutine" : "100.00", + "total" : "86.58" + } + } + }, + "0.30" : { + "coverage" : { + "total" : { + "branch" : "78.95", + "condition" : "46.67", + "pod" : "100.00", + "statement" : "94.87", + "subroutine" : "100.00", + "total" : "86.30" + } + } + } + } +} \ No newline at end of file From 38d5eb79317e6a4c61e196da339f54275bf4c4b6 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 20:49:11 +0000 Subject: [PATCH 056/892] Split t/script/cover.t and t/server/controller/cover.t --- t/script/cover.t | 54 -------------------------------- t/server/controller/cover.t | 61 +++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 54 deletions(-) create mode 100644 t/server/controller/cover.t diff --git a/t/script/cover.t b/t/script/cover.t index 0b3a44e6c..dd82550a1 100644 --- a/t/script/cover.t +++ b/t/script/cover.t @@ -5,8 +5,6 @@ use lib 't/lib'; use Git::Helpers qw( checkout_root ); use MetaCPAN::Script::Cover (); use MetaCPAN::Script::Runner (); -use MetaCPAN::Server::Test qw( app GET test_psgi ); -use MetaCPAN::TestHelpers qw( decode_json_ok ); use Test::More; use URI (); @@ -19,56 +17,4 @@ $config->{'cover_url'} = "$file"; my $cover = MetaCPAN::Script::Cover->new_with_options($config); ok $cover->run, 'runs and returns true'; -my %expect = ( - 'Devel-GoFaster-0.000' => { - criteria => { - branch => '12.50', - condition => '0.00', - statement => '63.64', - subroutine => '71.43', - total => '46.51', - }, - distribution => 'Devel-GoFaster', - release => 'Devel-GoFaster-0.000', - url => 'http://cpancover.com/latest/Devel-GoFaster-0.000/index.html', - version => '0.000', - }, - 'Try-Tiny-0.27' => { - criteria => { - branch => '78.95', - condition => '46.67', - statement => '95.06', - subroutine => '100.00', - total => '86.58', - }, - distribution => 'Try-Tiny', - release => 'Try-Tiny-0.27', - url => 'http://cpancover.com/latest/Try-Tiny-0.27/index.html', - version => '0.27', - }, -); - -my $test = Plack::Test->create( app() ); - -for my $release ( keys %expect ) { - my $expected = $expect{$release}; - subtest "Check $release" => sub { - my $url = "/cover/$release"; - my $res = $test->request( GET $url ); - diag "GET $url"; - - # TRAVIS 5.18 - is( $res->code, 200, "code 200" ); - is( - $res->header('content-type'), - 'application/json; charset=utf-8', - 'Content-type' - ); - my $json = decode_json_ok($res); - - # TRAVIS 5.18 - is_deeply( $json, $expected, "$release cover summary roundtrip" ); - }; -} - done_testing(); diff --git a/t/server/controller/cover.t b/t/server/controller/cover.t new file mode 100644 index 000000000..512605661 --- /dev/null +++ b/t/server/controller/cover.t @@ -0,0 +1,61 @@ +use strict; +use warnings; +use lib 't/lib'; + +use MetaCPAN::Server::Test qw( app GET test_psgi ); +use MetaCPAN::TestHelpers qw( decode_json_ok ); +use Test::More; + +my %expect = ( + 'Devel-GoFaster-0.000' => { + criteria => { + branch => '12.50', + condition => '0.00', + statement => '63.64', + subroutine => '71.43', + total => '46.51', + }, + distribution => 'Devel-GoFaster', + release => 'Devel-GoFaster-0.000', + url => 'http://cpancover.com/latest/Devel-GoFaster-0.000/index.html', + version => '0.000', + }, + 'Try-Tiny-0.27' => { + criteria => { + branch => '78.95', + condition => '46.67', + statement => '95.06', + subroutine => '100.00', + total => '86.58', + }, + distribution => 'Try-Tiny', + release => 'Try-Tiny-0.27', + url => 'http://cpancover.com/latest/Try-Tiny-0.27/index.html', + version => '0.27', + }, +); + +my $test = Plack::Test->create( app() ); + +for my $release ( keys %expect ) { + my $expected = $expect{$release}; + subtest "Check $release" => sub { + my $url = "/cover/$release"; + my $res = $test->request( GET $url ); + diag "GET $url"; + + # TRAVIS 5.18 + is( $res->code, 200, "code 200" ); + is( + $res->header('content-type'), + 'application/json; charset=utf-8', + 'Content-type' + ); + my $json = decode_json_ok($res); + + # TRAVIS 5.18 + is_deeply( $json, $expected, "$release cover summary roundtrip" ); + }; +} + +done_testing(); From b1e6de5fee631c68d9078ed3831c1e240edc0da6 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 22:06:04 +0000 Subject: [PATCH 057/892] Anchor cover test run order --- t/testrules.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/t/testrules.yml b/t/testrules.yml index 00a2812f0..3534e9659 100644 --- a/t/testrules.yml +++ b/t/testrules.yml @@ -2,6 +2,10 @@ seq: - seq: t/0*.t + # ensure t/script/cover.t runs before t/server/controller/cover.t + + - seq: t/script/cover.t + # If t/server/controller/user/favorite.t this runs too late then the # looks_human test will fail. We should probably reset the user data, but # this is a quicker fix for now. From da7b4a3ab26fc1b78ba39da4cfd36476343d0796 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 22:18:56 +0000 Subject: [PATCH 058/892] Fix /search/web test --- t/model/search.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/model/search.t b/t/model/search.t index a6dc2eb71..87c5ad516 100644 --- a/t/model/search.t +++ b/t/model/search.t @@ -78,7 +78,7 @@ ok( $search->_not_rogue, '_not_rogue' ); my $module = 'Binary::Data::WithPod'; my $results = $search->search_web($module); is( - $results->{results}->[0]->[0]->{description}, + $results->{results}->[0]->{hits}->[0]->{description}, 'razzberry pudding', 'description included in results' ); From d79b8b70868c06bdcd86d884615a1a116dbe3394 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Fri, 9 Nov 2018 10:53:28 -0600 Subject: [PATCH 059/892] Initial commit Start work on documenting the search API with OpenAPI. --- root/static/definitions/definitions.yml | 5 + root/static/definitions/parameters.yml | 12 ++ root/static/definitions/results.yml | 67 +++++++++ root/static/index.html | 24 ++++ root/static/v1.yml | 178 ++++++++++++++++++++++++ 5 files changed, 286 insertions(+) create mode 100644 root/static/definitions/definitions.yml create mode 100644 root/static/definitions/parameters.yml create mode 100644 root/static/definitions/results.yml create mode 100644 root/static/index.html create mode 100644 root/static/v1.yml diff --git a/root/static/definitions/definitions.yml b/root/static/definitions/definitions.yml new file mode 100644 index 000000000..b292922fa --- /dev/null +++ b/root/static/definitions/definitions.yml @@ -0,0 +1,5 @@ +--- + +# Maintain names and descriptions of common attributes +dist_fav_count: + description: Number of times favorited diff --git a/root/static/definitions/parameters.yml b/root/static/definitions/parameters.yml new file mode 100644 index 000000000..85fee13e6 --- /dev/null +++ b/root/static/definitions/parameters.yml @@ -0,0 +1,12 @@ +--- + +q: + name: q + description: | + The query search term. If the search term contains a term with the + tags `dist:` or `module:` results will be in expanded form, otherwise + collapsed form. + + See also `collapsed` + type: string + required: true diff --git a/root/static/definitions/results.yml b/root/static/definitions/results.yml new file mode 100644 index 000000000..e8c628c8b --- /dev/null +++ b/root/static/definitions/results.yml @@ -0,0 +1,67 @@ +--- + +search_result_item: + type: object + properties: + description: + type: string + description: + documentation: + type: string + description: + authorized: + type: boolean + path: + type: string + author: + type: string + id: + type: string + date: + type: string + favorites: + type: integer + status: + type: string + score: + type: number + module: + type: array + items: + type: object + properties: + associated_pod: + type: string + indexed: + type: boolean + name: + type: string + authorized: + type: boolean + version_numified: + type: number + distribution: + type: string + indexed: + type: boolean, + pod_lines: + type: array + abstract: + type: string + release: + type: string +dependency: + type: object + properties: + module: + type: string + # "Mojolicious", + phase: + type: string + # "runtime", + version: + type: string + # "8", + relationship: + type: string + # "requires" diff --git a/root/static/index.html b/root/static/index.html new file mode 100644 index 000000000..00d8388cf --- /dev/null +++ b/root/static/index.html @@ -0,0 +1,24 @@ + + + + ReDoc + + + + + + + + + + + + + diff --git a/root/static/v1.yml b/root/static/v1.yml new file mode 100644 index 000000000..6f5186cc4 --- /dev/null +++ b/root/static/v1.yml @@ -0,0 +1,178 @@ +--- + +swagger: "2.0" +info: + version: "1.0" + title: "MetaCPAN API" +basePath: "/v1" +tags: + - Search: + name: Search + description: MetaCPAN Search Endpoints + - Release: + name: Release + description: Distribution Release Endpoints +paths: + /search/web: + get: + tags: + - Search + operationId: search_web + summary: Perform API search in the same fashion as the Web UI + parameters: + - in: query + $ref: "./definitions/parameters.yml#/q" + - in: query + name: from + description: The offset to use in the result set + type: integer + default: 0 + - in: query + name: page_size + description: Number of results per page + type: integer + default: 20 + - in: query + name: collapsed + description: | + Force a collapsed even when searching for a particular + distribution or module name. + type: boolean + responses: + 200: + description: Search response + schema: + type: object + properties: + total: + type: integer + took: + collapsed: + results: + title: Results + type: array + items: + type: array + items: + $ref: "./definitions/results.yml#/search_result_item" + /search/first: + get: + tags: + - Search + operationId: search_for_first + summary: Perform API search and return the first result (I'm Feeling Lucky) + parameters: + - in: query + name: q + description: | + The query search term. + type: string + required: true + responses: + 200: + description: Search response + schema: + type: object + properties: + path: + type: string + description: Relative path to module with full name + # "lib/Moo.pm", + authorized: + type: boolean + # true, + description: + type: string + description: Module description + # "\"Moo\" is an extremely light-weight Object Orientation system. It allows one to concisely define objects and roles with a convenient syntax that avoids the details of Perl's object system. \"Moo\" contains a subset of Moose and is optimised for rapid startup. \"Moo\" avoids depending on any XS modules to allow for simple deployments. The name \"Moo\" is based on the idea that it provides almost -- but not quite -- two thirds of Moose. Unlike Mouse this module does not aim at full compatibility with Moose's surface syntax, preferring instead to provide full interoperability via the metaclass inflation capabilities described in \"MOO AND MOOSE\". For a full list of the minor differences between Moose and Moo's surface syntax, see \"INCOMPATIBILITIES WITH MOOSE\".", + id: + type: string + # "xnl1_tvOXN5leFY9xIUOWrAiRso", + distribution: + type: string + description: Name of the distribution the module is contained in + # "Moo", + author: + type: string + description: Module author ID + # "HAARG", + release: + type: string + description: Package name with version + status: + type: string + # "latest", + abstract.analyzed: + type: string + description: The module's abstract as analyzed from POD + # "Minimalist Object Orientation (with Moose compatibility)", + dist_fav_count: + # $ref: "./definitions/definitions.yml#/dist_fav_count" + type: integer + description: Number of times favorited + # 258, + date: + type: string + description: date module was indexed + # "2017-12-01T01:48:32", + documentation: + type: string + # "Moo", + pod_lines: + type: array + items: + type: integer + # [ + # 254, + # 829 + # ], + indexed: + type: boolean + description: Is the module indexed by PAUSE + # true + /release/recent: + get: + tags: + - Release + operationId: release_recent + summary: Get recent releases + parameters: + - in: path + name: name + description: | + The name of the Release + type: string + required: true + responses: + 200: + description: Release response + schema: + type: object + properties: + name: + type: string + dependency: + type: array + items: + $ref: "./definitions/results.yml#/dependency" + /release/{name}: + get: + tags: + - Release + operationId: release_by_name + summary: Get details about a release + parameters: + - in: path + name: name + description: | + The name of the Release + type: string + required: true + responses: + 200: + description: Release response + schema: + type: object + properties: + name: + type: string From f9e2f77631cedee0b2523a0626c88ea1d9730d23 Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Thu, 8 Nov 2018 21:17:35 -0600 Subject: [PATCH 060/892] add the ability to run the api from the mojo app --- cpanfile | 1 + lib/MetaCPAN/Queue.pm | 5 ++++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/cpanfile b/cpanfile index 004e3dc51..5c0aac9ab 100644 --- a/cpanfile +++ b/cpanfile @@ -176,6 +176,7 @@ requires 'strictures', 1; requires 'utf8'; requires 'version', '0.9901'; requires 'warnings'; +requires 'Mojolicious::Plugin::MountPSGI'; test_requires 'App::Prove'; test_requires 'CPAN::Faker', '0.010'; diff --git a/lib/MetaCPAN/Queue.pm b/lib/MetaCPAN/Queue.pm index c19f6e538..da2d24791 100644 --- a/lib/MetaCPAN/Queue.pm +++ b/lib/MetaCPAN/Queue.pm @@ -32,7 +32,10 @@ sub startup { my $helper = MetaCPAN::Queue::Helper->new; $self->plugin( Minion => $helper->backend ); - $self->plugin( 'Minion::Admin' => { route => $self->routes->any('/') } ); + $self->plugin( + 'Minion::Admin' => { route => $self->routes->any('/minion') } ); + $self->plugin( + MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); $self->minion->add_task( index_release => $self->_gen_index_task_sub('release') ); From adf07ca6fec672cf85822c8fda0236571ea09d7e Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Fri, 9 Nov 2018 18:55:00 +0000 Subject: [PATCH 061/892] Move MetaCPAN::Queue to MetaCPAN::API --- bin/{queue.pl => api.pl} | 2 +- lib/MetaCPAN/{Queue.pm => API.pm} | 26 ++++++++++++++++++----- lib/MetaCPAN/Queue/Helper.pm | 35 ------------------------------- lib/MetaCPAN/Role/Script.pm | 4 ++-- t/queue.t | 8 +++++-- t/queue/helper.t | 12 ----------- 6 files changed, 30 insertions(+), 57 deletions(-) rename bin/{queue.pl => api.pl} (90%) rename lib/MetaCPAN/{Queue.pm => API.pm} (75%) delete mode 100644 lib/MetaCPAN/Queue/Helper.pm delete mode 100644 t/queue/helper.t diff --git a/bin/queue.pl b/bin/api.pl similarity index 90% rename from bin/queue.pl rename to bin/api.pl index dad5619c0..7fd74becd 100755 --- a/bin/queue.pl +++ b/bin/api.pl @@ -28,4 +28,4 @@ =head2 DESCRIPTION # Start command line interface for application require Mojolicious::Commands; -Mojolicious::Commands->start_app('MetaCPAN::Queue'); +Mojolicious::Commands->start_app('MetaCPAN::API'); diff --git a/lib/MetaCPAN/Queue.pm b/lib/MetaCPAN/API.pm similarity index 75% rename from lib/MetaCPAN/Queue.pm rename to lib/MetaCPAN/API.pm index da2d24791..f752e2284 100644 --- a/lib/MetaCPAN/Queue.pm +++ b/lib/MetaCPAN/API.pm @@ -1,4 +1,4 @@ -package MetaCPAN::Queue; +package MetaCPAN::API; =head1 DESCRIPTION @@ -20,18 +20,34 @@ To run the minion admin web interface, run the following on one of the servers: use Mojo::Base 'Mojolicious'; -use MetaCPAN::Queue::Helper (); +use Config::ZOMG (); +use File::Temp (); use MetaCPAN::Script::Runner (); use Try::Tiny qw( catch try ); sub startup { my $self = shift; - # for Mojo cookies, which we won't be needing + unless ( $self->config->{config_override} ) { + $self->config( + Config::ZOMG->new( + name => 'metacpan_server', + path => $self->home->to_string, + )->load + ); + } + + # TODO secret from config $self->secrets( ['veni vidi vici'] ); - my $helper = MetaCPAN::Queue::Helper->new; - $self->plugin( Minion => $helper->backend ); + if ( $ENV{HARNESS_ACTIVE} ) { + my $file = File::Temp->new( UNLINK => 1, SUFFIX => '.db' ); + $self->plugin( Minion => { SQLite => 'sqlite:' . $file } ); + } + else { + $self->plugin( Minion => { Pg => $self->config->{minion_dsn} } ); + } + $self->plugin( 'Minion::Admin' => { route => $self->routes->any('/minion') } ); $self->plugin( diff --git a/lib/MetaCPAN/Queue/Helper.pm b/lib/MetaCPAN/Queue/Helper.pm deleted file mode 100644 index 975356d8c..000000000 --- a/lib/MetaCPAN/Queue/Helper.pm +++ /dev/null @@ -1,35 +0,0 @@ -package MetaCPAN::Queue::Helper; - -use Moose; - -use File::Temp (); -use MetaCPAN::Types qw( HashRef ); -use Module::Load qw( load ); - -has backend => ( - is => 'ro', - isa => HashRef, - lazy => 1, - builder => '_build_backend', -); - -with 'MetaCPAN::Role::HasConfig'; - -# We could also use an in-memory SQLite db, but this gives us the option of not -# unlinking in order to debug the contents of the db, if we need to. - -sub _build_backend { - my $self = shift; - - if ( $ENV{HARNESS_ACTIVE} ) { - load(Minion::Backend::SQLite); - my $file = File::Temp->new( UNLINK => 1, SUFFIX => '.db' ); - return { SQLite => 'sqlite:' . $file }; - } - - load(Minion::Backend::Pg); - return { Pg => $self->config->{minion_dsn} }; -} - -__PACKAGE__->meta->make_immutable; -1; diff --git a/lib/MetaCPAN/Role/Script.pm b/lib/MetaCPAN/Role/Script.pm index 9657573dd..43c6195e6 100644 --- a/lib/MetaCPAN/Role/Script.pm +++ b/lib/MetaCPAN/Role/Script.pm @@ -8,7 +8,7 @@ use Git::Helpers qw( checkout_root ); use Log::Contextual qw( :log :dlog ); use MetaCPAN::Model; use MetaCPAN::Types qw(:all); -use MetaCPAN::Queue (); +use Mojo::Server; use Term::ANSIColor qw( colored ); use IO::Interactive qw( is_interactive ); use IO::Prompt; @@ -99,7 +99,7 @@ has _minion => ( isa => 'Minion', lazy => 1, handles => { _add_to_queue => 'enqueue', stats => 'stats', }, - default => sub { MetaCPAN::Queue->new->minion }, + default => sub { Mojo::Server->new->build_app('MetaCPAN::API')->minion }, ); has queue => ( diff --git a/t/queue.t b/t/queue.t index 848ec9838..1f180ee88 100644 --- a/t/queue.t +++ b/t/queue.t @@ -2,12 +2,16 @@ use strict; use warnings; use lib 't/lib'; -use MetaCPAN::Queue; use Test::More; use Test::RequiresInternet ( 'cpan.metacpan.org' => 443 ); -my $app = MetaCPAN::Queue->new; +use Test::Mojo; + +my $t = Test::Mojo->new('MetaCPAN::API'); +my $app = $t->app; + ok( $app, 'queue app' ); +isa_ok $app, 'MetaCPAN::API'; my $release = 'https://cpan.metacpan.org/authors/id/O/OA/OALDERS/HTML-Restrict-2.2.2.tar.gz'; diff --git a/t/queue/helper.t b/t/queue/helper.t deleted file mode 100644 index d7205662a..000000000 --- a/t/queue/helper.t +++ /dev/null @@ -1,12 +0,0 @@ -use strict; -use warnings; -use lib 't/lib'; - -use MetaCPAN::Queue::Helper; -use Test::More; - -my $helper = MetaCPAN::Queue::Helper->new; - -ok( $helper->backend, 'backend' ); - -done_testing(); From 88fced0a3cb0282d40d7a9e5fd0c32afdc9b903e Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Fri, 9 Nov 2018 19:43:06 +0000 Subject: [PATCH 062/892] load es and model_search in the main application --- lib/MetaCPAN/API.pm | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index f752e2284..2728c6591 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -22,9 +22,26 @@ use Mojo::Base 'Mojolicious'; use Config::ZOMG (); use File::Temp (); +use MetaCPAN::Model::Search (); use MetaCPAN::Script::Runner (); +use Search::Elasticsearch (); use Try::Tiny qw( catch try ); +has es => sub { + return Search::Elasticsearch->new( + client => '2_0::Direct', + nodes => [':9200'], #TODO config + ); +}; + +has model_search => sub { + my $self = shift; + return MetaCPAN::Model::Search->new( + es => $self->es, + index => 'cpan', + ); +}; + sub startup { my $self = shift; From dd5afd5806c1a8ab3e39dcd85ec664c657ab6447 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 8 Nov 2018 16:12:06 -0600 Subject: [PATCH 063/892] Add Mojolicious::Plugin::Web::Auth to carton --- cpanfile | 1 + cpanfile.snapshot | 20 ++++++++++++++++++++ 2 files changed, 21 insertions(+) diff --git a/cpanfile b/cpanfile index 5c0aac9ab..aecb3c2a8 100644 --- a/cpanfile +++ b/cpanfile @@ -100,6 +100,7 @@ requires 'Module::Metadata', '1.000022'; requires 'Module::Pluggable'; requires 'Module::Runtime'; requires 'Mojo::Pg', '>= 4.08'; +requires 'Mojolicious::Plugin::Web::Auth', '0.000004'; requires 'Moose', ' >= 2.1403'; requires 'Moose::Role'; requires 'Moose::Util'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 9dff276ef..c02e8ca3e 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4842,6 +4842,26 @@ DISTRIBUTIONS Pod::Simple 3.09 Time::Local 1.2 perl 5.010001 + Mojolicious-Plugin-Web-Auth-0.15 + pathname: H/HA/HAYAJO/Mojolicious-Plugin-Web-Auth-0.15.tar.gz + provides: + Mojolicious::Plugin::Web::Auth 0.15 + Mojolicious::Plugin::Web::Auth::Base undef + Mojolicious::Plugin::Web::Auth::OAuth undef + Mojolicious::Plugin::Web::Auth::OAuth2 undef + Mojolicious::Plugin::Web::Auth::Site::Dropbox undef + Mojolicious::Plugin::Web::Auth::Site::Facebook undef + Mojolicious::Plugin::Web::Auth::Site::Github undef + Mojolicious::Plugin::Web::Auth::Site::Google undef + Mojolicious::Plugin::Web::Auth::Site::Instagram undef + Mojolicious::Plugin::Web::Auth::Site::Twitter undef + Mojolicious::Plugin::Web::Auth::Site::Yandex undef + requirements: + IO::Socket::SSL 1.77 + Module::Build::Tiny 0.035 + Mojolicious 3.02 + Net::OAuth 0.28 + perl 5.008005 Moo-2.003003 pathname: H/HA/HAARG/Moo-2.003003.tar.gz provides: From 8f569a8733a62450507b41097bc1c8e6203b0f1b Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 8 Nov 2018 21:12:21 -0600 Subject: [PATCH 064/892] Mount an admin app at /admin Add a Mojo admin controller Rename MetaCPAN::Queue to MetaCPAN::Admin Queue test doesn't require internet Queue test doesn't need network access Rename Mojo controllers Add Mojo templates --- app.psgi | 10 +-- lib/MetaCPAN/API.pm | 76 ++++++++++++++++++-- lib/MetaCPAN/API/Controller/Admin.pm | 8 +++ lib/MetaCPAN/API/Controller/Queue.pm | 19 +++++ t/admin.t | 37 ++++++++++ t/queue.t | 8 +-- templates/admin/identity_search_form.html.ep | 14 ++++ templates/admin/index.html.ep | 8 +++ templates/admin/search_identities.html.ep | 1 + templates/layouts/default.html.ep | 5 ++ templates/queue/index_release.html.ep | 0 11 files changed, 171 insertions(+), 15 deletions(-) create mode 100644 lib/MetaCPAN/API/Controller/Admin.pm create mode 100644 lib/MetaCPAN/API/Controller/Queue.pm create mode 100644 t/admin.t create mode 100644 templates/admin/identity_search_form.html.ep create mode 100644 templates/admin/index.html.ep create mode 100644 templates/admin/search_identities.html.ep create mode 100644 templates/layouts/default.html.ep create mode 100644 templates/queue/index_release.html.ep diff --git a/app.psgi b/app.psgi index 456c84e42..ac05e1f6e 100644 --- a/app.psgi +++ b/app.psgi @@ -1,11 +1,11 @@ use strict; use warnings; -use Config::ZOMG (); -use File::Basename (); -use File::Path (); -use File::Spec (); -use Log::Log4perl (); +use Config::ZOMG (); +use File::Basename (); +use File::Path (); +use File::Spec (); +use Log::Log4perl (); use Path::Tiny qw( path ); use Plack::App::Directory (); use Plack::App::URLMap (); diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 2728c6591..4ffc540e8 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -22,6 +22,7 @@ use Mojo::Base 'Mojolicious'; use Config::ZOMG (); use File::Temp (); +use List::Util qw( any ); use MetaCPAN::Model::Search (); use MetaCPAN::Script::Runner (); use Search::Elasticsearch (); @@ -55,7 +56,8 @@ sub startup { } # TODO secret from config - $self->secrets( ['veni vidi vici'] ); + $self->secrets( [ $ENV{MOJO_SECRET} ] ); + if ( $ENV{HARNESS_ACTIVE} ) { my $file = File::Temp->new( UNLINK => 1, SUFFIX => '.db' ); @@ -65,11 +67,6 @@ sub startup { $self->plugin( Minion => { Pg => $self->config->{minion_dsn} } ); } - $self->plugin( - 'Minion::Admin' => { route => $self->routes->any('/minion') } ); - $self->plugin( - MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); - $self->minion->add_task( index_release => $self->_gen_index_task_sub('release') ); @@ -78,6 +75,8 @@ sub startup { $self->minion->add_task( index_favorite => $self->_gen_index_task_sub('favorite') ); + + $self->_maybe_set_up_routes; } sub _gen_index_task_sub { @@ -113,4 +112,69 @@ sub _gen_index_task_sub { } } +sub _maybe_set_up_routes { + my $self = shift; + return unless $ENV{MOJO_SECRET} && $ENV{GITHUB_KEY}; + + my $r = $self->routes; + + $self->plugin( + 'Web::Auth', + module => 'Github', + key => $ENV{GITHUB_KEY}, + secret => $ENV{GITHUB_SECRET}, + on_finished => sub { + my ( $c, $access_token, $account_info ) = @_; + my $login = $account_info->{login}; + if ( $self->_is_admin($login) ) { + $c->session( username => $login ); + $c->redirect_to('/admin'); + return; + } + return $c->render( + text => "$login is not authorized to access this application", + status => 403 + ); + }, + ); + + my $admin = $r->under( + '/admin' => sub { + my $c = shift; + return 1 if $self->_is_admin( $c->session('username') ); + $c->redirect_to('/auth/github/authenticate'); + return 0; + } + ); + + $admin->get('home')->to('admin#home')->name('admin-home'); + $admin->post('enqueue')->to('queue#enqueue')->name('enqueue'); + $admin->post('search-identities')->to('admin#search_identities') + ->name('search-identities'); + $admin->get('index-release')->to('queue#index_release') + ->name('index-release'); + $admin->get('identity-search-form')->to('admin#identity_search_form') + ->name('identity_search_form'); + + $self->plugin( 'Minion::Admin' => { route => $admin->any('/minion') } ); + $self->plugin( + MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); + +} + +sub _is_admin { + my $self = shift; + my $username + = shift || ( $ENV{HARNESS_ACTIVE} ? $ENV{FORCE_ADMIN_AUTH} : () ); + return 0 unless $username; + + my @admins = ( + 'haarg', 'jberger', 'mickeyn', 'oalders', + 'ranguard', 'reyjrar', 'ssoriche', + $ENV{HARNESS_ACTIVE} ? 'tester' : (), + ); + + return any { $username eq $_ } @admins; +} + 1; diff --git a/lib/MetaCPAN/API/Controller/Admin.pm b/lib/MetaCPAN/API/Controller/Admin.pm new file mode 100644 index 000000000..6c6a5fb8c --- /dev/null +++ b/lib/MetaCPAN/API/Controller/Admin.pm @@ -0,0 +1,8 @@ +package MetaCPAN::API::Controller::Admin; + +use Mojo::Base 'Mojolicious::Controller'; + +sub identity_search_form { } +sub search_identities { } + +1; diff --git a/lib/MetaCPAN/API/Controller/Queue.pm b/lib/MetaCPAN/API/Controller/Queue.pm new file mode 100644 index 000000000..d35c66167 --- /dev/null +++ b/lib/MetaCPAN/API/Controller/Queue.pm @@ -0,0 +1,19 @@ +package MetaCPAN::API::Controller::Queue; + +use Mojo::Base 'Mojolicious::Controller'; + +my $rel + = 'https://cpan.metacpan.org/authors/id/O/OA/OALDERS/HTML-Restrict-2.2.2.tar.gz'; + +sub enqueue { + my $self = shift; + $self->minion->enqueue( index_release => [ '--latest', $rel ] ); + $self->render( text => 'OK' ); +} + +sub index_release { + my $self = shift; + $self->render( text => 'ok' ); +} + +1; diff --git a/t/admin.t b/t/admin.t new file mode 100644 index 000000000..b720c8ab4 --- /dev/null +++ b/t/admin.t @@ -0,0 +1,37 @@ +use strict; +use warnings; +use lib 't/lib'; + +use Test::Fatal qw( exception ); +use Test::Mojo; +use Test::More; + +local $ENV{MOJO_SECRET} = 'Magritte'; +local $ENV{GITHUB_KEY} = 'foo'; +local $ENV{GITHUB_SECRET} = 'bar'; + +subtest 'authentication enabled' => sub { + my $t = Test::Mojo->new('MetaCPAN::API'); + $t->post_ok('/admin/enqueue'); + $t->header_is( Location => '/auth/github/authenticate' ); + $t->status_is(302); +}; + +subtest 'index release' => sub { + local $ENV{FORCE_ADMIN_AUTH} = 'tester'; + my $t = Test::Mojo->new('MetaCPAN::API'); + $t->get_ok('/admin/index-release'); + $t->status_is(200); +}; + +subtest 'search identities' => sub { + local $ENV{FORCE_ADMIN_AUTH} = 'tester'; + my $t = Test::Mojo->new('MetaCPAN::API'); + $t->get_ok('/admin/identity-search-form'); + $t->status_is(200); + + $t->post_ok('/admin/search-identities'); + $t->status_is(200); +}; + +done_testing(); diff --git a/t/queue.t b/t/queue.t index 1f180ee88..5a9f747ea 100644 --- a/t/queue.t +++ b/t/queue.t @@ -2,9 +2,9 @@ use strict; use warnings; use lib 't/lib'; +use MetaCPAN::DarkPAN (); +use Path::Tiny qw( path ); use Test::More; -use Test::RequiresInternet ( 'cpan.metacpan.org' => 443 ); - use Test::Mojo; my $t = Test::Mojo->new('MetaCPAN::API'); @@ -13,8 +13,8 @@ my $app = $t->app; ok( $app, 'queue app' ); isa_ok $app, 'MetaCPAN::API'; -my $release - = 'https://cpan.metacpan.org/authors/id/O/OA/OALDERS/HTML-Restrict-2.2.2.tar.gz'; +my $darkpan = MetaCPAN::DarkPAN->new->base_dir; +my $release = path( $darkpan, 'authors/id/E/ET/ETHER/Try-Tiny-0.23.tar.gz' ); $app->minion->enqueue( index_release => [$release] ); $app->minion->enqueue( index_release => [ '--latest', $release ] ); diff --git a/templates/admin/identity_search_form.html.ep b/templates/admin/identity_search_form.html.ep new file mode 100644 index 000000000..06bcfb737 --- /dev/null +++ b/templates/admin/identity_search_form.html.ep @@ -0,0 +1,14 @@ +
+ + + + Identity value: + + +
diff --git a/templates/admin/index.html.ep b/templates/admin/index.html.ep new file mode 100644 index 000000000..a39edf7f1 --- /dev/null +++ b/templates/admin/index.html.ep @@ -0,0 +1,8 @@ + diff --git a/templates/admin/search_identities.html.ep b/templates/admin/search_identities.html.ep new file mode 100644 index 000000000..da0175af0 --- /dev/null +++ b/templates/admin/search_identities.html.ep @@ -0,0 +1 @@ +display results below: diff --git a/templates/layouts/default.html.ep b/templates/layouts/default.html.ep new file mode 100644 index 000000000..fbf9c181c --- /dev/null +++ b/templates/layouts/default.html.ep @@ -0,0 +1,5 @@ +

+
MetaCPAN Admin
+

+ +<%= content %> diff --git a/templates/queue/index_release.html.ep b/templates/queue/index_release.html.ep new file mode 100644 index 000000000..e69de29bb From e007dc9648869f441b191c532f76269f83690183 Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Fri, 9 Nov 2018 21:43:12 +0000 Subject: [PATCH 065/892] allow mojo to serve the static assets from root/ --- app.psgi | 10 +++++----- lib/MetaCPAN/API.pm | 5 +++-- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/app.psgi b/app.psgi index ac05e1f6e..456c84e42 100644 --- a/app.psgi +++ b/app.psgi @@ -1,11 +1,11 @@ use strict; use warnings; -use Config::ZOMG (); -use File::Basename (); -use File::Path (); -use File::Spec (); -use Log::Log4perl (); +use Config::ZOMG (); +use File::Basename (); +use File::Path (); +use File::Spec (); +use Log::Log4perl (); use Path::Tiny qw( path ); use Plack::App::Directory (); use Plack::App::URLMap (); diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 4ffc540e8..acb8007a5 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -20,8 +20,8 @@ To run the minion admin web interface, run the following on one of the servers: use Mojo::Base 'Mojolicious'; -use Config::ZOMG (); -use File::Temp (); +use Config::ZOMG (); +use File::Temp (); use List::Util qw( any ); use MetaCPAN::Model::Search (); use MetaCPAN::Script::Runner (); @@ -58,6 +58,7 @@ sub startup { # TODO secret from config $self->secrets( [ $ENV{MOJO_SECRET} ] ); + $self->static->paths( [ $self->home->child('root') ] ); if ( $ENV{HARNESS_ACTIVE} ) { my $file = File::Temp->new( UNLINK => 1, SUFFIX => '.db' ); From d2957cdf2b739824fddee1ea104780caeaa0bfb6 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 15:55:35 -0600 Subject: [PATCH 066/892] Add and update Mojo deps in cpanfile.snapshot --- cpanfile.snapshot | 34 +++++++++++++++++++++------------- 1 file changed, 21 insertions(+), 13 deletions(-) diff --git a/cpanfile.snapshot b/cpanfile.snapshot index c02e8ca3e..2c8c206f9 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4720,8 +4720,8 @@ DISTRIBUTIONS URI::db 0.15 URI::file 4.21 perl 5.010001 - Mojolicious-7.56 - pathname: S/SR/SRI/Mojolicious-7.56.tar.gz + Mojolicious-8.06 + pathname: S/SR/SRI/Mojolicious-8.06.tar.gz provides: Mojo undef Mojo::Asset undef @@ -4741,6 +4741,7 @@ DISTRIBUTIONS Mojo::DOM::CSS undef Mojo::DOM::HTML undef Mojo::Date undef + Mojo::DynamicMethods undef Mojo::EventEmitter undef Mojo::Exception undef Mojo::File undef @@ -4790,30 +4791,28 @@ DISTRIBUTIONS Mojo::UserAgent::Transactor undef Mojo::Util undef Mojo::WebSocket undef - Mojolicious 7.56 + Mojolicious 8.06 Mojolicious::Command undef + Mojolicious::Command::Author::cpanify undef + Mojolicious::Command::Author::generate undef + Mojolicious::Command::Author::generate::app undef + Mojolicious::Command::Author::generate::lite_app undef + Mojolicious::Command::Author::generate::makefile undef + Mojolicious::Command::Author::generate::plugin undef + Mojolicious::Command::Author::inflate undef Mojolicious::Command::cgi undef - Mojolicious::Command::cpanify undef Mojolicious::Command::daemon undef Mojolicious::Command::eval undef - Mojolicious::Command::generate undef - Mojolicious::Command::generate::app undef - Mojolicious::Command::generate::lite_app undef - Mojolicious::Command::generate::makefile undef - Mojolicious::Command::generate::plugin undef Mojolicious::Command::get undef - Mojolicious::Command::inflate undef Mojolicious::Command::prefork undef Mojolicious::Command::psgi undef Mojolicious::Command::routes undef - Mojolicious::Command::test undef Mojolicious::Command::version undef Mojolicious::Commands undef Mojolicious::Controller undef Mojolicious::Lite undef Mojolicious::Plugin undef Mojolicious::Plugin::Config undef - Mojolicious::Plugin::Config::Sandbox undef Mojolicious::Plugin::DefaultHelpers undef Mojolicious::Plugin::EPLRenderer undef Mojolicious::Plugin::EPRenderer undef @@ -4839,9 +4838,18 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 IO::Socket::IP 0.37 JSON::PP 2.27103 - Pod::Simple 3.09 + List::Util 1.41 Time::Local 1.2 perl 5.010001 + Mojolicious-Plugin-MountPSGI-0.13 + pathname: M/MR/MRAMBERG/Mojolicious-Plugin-MountPSGI-0.13.tar.gz + provides: + Mojolicious::Plugin::MountPSGI 0.13 + Mojolicious::Plugin::MountPSGI::Proxy undef + requirements: + ExtUtils::MakeMaker 0 + Mojolicious 7.70 + Plack 0 Mojolicious-Plugin-Web-Auth-0.15 pathname: H/HA/HAYAJO/Mojolicious-Plugin-Web-Auth-0.15.tar.gz provides: From b0d65b9661e7d241fa3ec3217960e9e33a0b53df Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Fri, 9 Nov 2018 23:32:50 +0000 Subject: [PATCH 067/892] start using the mojo application for routes in the openapi spec --- cpanfile | 2 + cpanfile.snapshot | 73 +++++++++++++++++++++----- lib/MetaCPAN/API.pm | 2 + lib/MetaCPAN/API/Controller/Search.pm | 24 +++++++++ root/static/definitions/parameters.yml | 12 ----- root/static/definitions/results.yml | 22 ++++---- root/static/v1.yml | 24 ++++++--- 7 files changed, 115 insertions(+), 44 deletions(-) create mode 100644 lib/MetaCPAN/API/Controller/Search.pm delete mode 100644 root/static/definitions/parameters.yml diff --git a/cpanfile b/cpanfile index aecb3c2a8..df6120a0d 100644 --- a/cpanfile +++ b/cpanfile @@ -178,6 +178,8 @@ requires 'utf8'; requires 'version', '0.9901'; requires 'warnings'; requires 'Mojolicious::Plugin::MountPSGI'; +requires 'Mojolicious::Plugin::OpenAPI'; +requires 'YAML::XS', '0.67'; # Mojolicious::Plugin::OpenAPI YAML loading test_requires 'App::Prove'; test_requires 'CPAN::Faker', '0.010'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 2c8c206f9..44b0cfa03 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -813,6 +813,7 @@ DISTRIBUTIONS MooseX::Emulate::Class::Accessor::Fast 0.00903 MooseX::Getopt 0.48 MooseX::MethodAttributes::Role::AttrContainer::Inheritable 0.24 + MooseX::Role::WithOverloading 0.09 Path::Class 0.09 Plack 0.9991 Plack::Middleware::Conditional 0 @@ -3821,6 +3822,19 @@ DISTRIBUTIONS JSON::PP 2.27300 Scalar::Util 0 perl 5.006 + JSON-Validator-2.15 + pathname: J/JH/JHTHORSEN/JSON-Validator-2.15.tar.gz + provides: + JSON::Validator 2.15 + JSON::Validator::Error undef + JSON::Validator::Joi undef + JSON::Validator::OpenAPI undef + JSON::Validator::OpenAPI::Dancer2 undef + JSON::Validator::OpenAPI::Mojolicious undef + JSON::Validator::Ref undef + requirements: + ExtUtils::MakeMaker 0 + Mojolicious 7.28 JSON-XS-3.04 pathname: M/ML/MLEHMANN/JSON-XS-3.04.tar.gz provides: @@ -4281,17 +4295,6 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.006 - Mac-SystemDirectory-0.10 - pathname: E/ET/ETHER/Mac-SystemDirectory-0.10.tar.gz - provides: - Mac::SystemDirectory 0.10 - requirements: - Exporter 0 - ExtUtils::MakeMaker 0 - XSLoader 0 - perl 5.006 - strict 0 - warnings 0 MailTools-2.19 pathname: M/MA/MARKOV/MailTools-2.19.tar.gz provides: @@ -4720,8 +4723,8 @@ DISTRIBUTIONS URI::db 0.15 URI::file 4.21 perl 5.010001 - Mojolicious-8.06 - pathname: S/SR/SRI/Mojolicious-8.06.tar.gz + Mojolicious-8.05 + pathname: S/SR/SRI/Mojolicious-8.05.tar.gz provides: Mojo undef Mojo::Asset undef @@ -4791,7 +4794,7 @@ DISTRIBUTIONS Mojo::UserAgent::Transactor undef Mojo::Util undef Mojo::WebSocket undef - Mojolicious 8.06 + Mojolicious 8.05 Mojolicious::Command undef Mojolicious::Command::Author::cpanify undef Mojolicious::Command::Author::generate undef @@ -4850,6 +4853,15 @@ DISTRIBUTIONS ExtUtils::MakeMaker 0 Mojolicious 7.70 Plack 0 + Mojolicious-Plugin-OpenAPI-2.01 + pathname: J/JH/JHTHORSEN/Mojolicious-Plugin-OpenAPI-2.01.tar.gz + provides: + Mojolicious::Plugin::OpenAPI 2.01 + Mojolicious::Plugin::OpenAPI::Cors undef + Mojolicious::Plugin::OpenAPI::Security undef + requirements: + ExtUtils::MakeMaker 0 + JSON::Validator 2.14 Mojolicious-Plugin-Web-Auth-0.15 pathname: H/HA/HAYAJO/Mojolicious-Plugin-Web-Auth-0.15.tar.gz provides: @@ -5540,6 +5552,30 @@ DISTRIBUTIONS perl 5.008001 strict 0 warnings 0 + MooseX-Role-WithOverloading-0.17 + pathname: E/ET/ETHER/MooseX-Role-WithOverloading-0.17.tar.gz + provides: + MooseX::Role::WithOverloading 0.17 + MooseX::Role::WithOverloading::Meta::Role 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::Composite 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::Composite::ToClass 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::Composite::ToInstance 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::Composite::ToRole 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::FixOverloadedRefs 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::ToClass 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::ToInstance 0.17 + MooseX::Role::WithOverloading::Meta::Role::Application::ToRole 0.17 + MooseX::Role::WithOverloading::Meta::Role::Composite 0.17 + requirements: + ExtUtils::MakeMaker 0 + Moose 0.94 + Moose::Exporter 0 + Moose::Role 1.15 + aliased 0 + namespace::autoclean 0.16 + namespace::clean 0.19 + perl 5.006 MooseX-StrictConstructor-0.21 pathname: D/DR/DROLSKY/MooseX-StrictConstructor-0.21.tar.gz provides: @@ -9214,6 +9250,15 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 perl 5.008001 + YAML-LibYAML-0.75 + pathname: T/TI/TINITA/YAML-LibYAML-0.75.tar.gz + provides: + YAML::LibYAML 0.75 + YAML::XS 0.75 + YAML::XS::LibYAML undef + requirements: + ExtUtils::MakeMaker 0 + perl 5.008001 YAML-Syck-1.30 pathname: T/TO/TODDR/YAML-Syck-1.30.tar.gz provides: diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index acb8007a5..ace32277a 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -158,6 +158,8 @@ sub _maybe_set_up_routes { ->name('identity_search_form'); $self->plugin( 'Minion::Admin' => { route => $admin->any('/minion') } ); + $self->plugin( + 'OpenAPI' => { url => $self->home->rel_file('root/static/v1.yml') } ); $self->plugin( MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); diff --git a/lib/MetaCPAN/API/Controller/Search.pm b/lib/MetaCPAN/API/Controller/Search.pm new file mode 100644 index 000000000..bb45970ca --- /dev/null +++ b/lib/MetaCPAN/API/Controller/Search.pm @@ -0,0 +1,24 @@ +package MetaCPAN::API::Controller::Search; + +use Mojo::Base 'Mojolicious::Controller'; + +sub web { + my $c = shift; + return unless $c->openapi->valid_input; + my $args = $c->validation->output; + + my @search = ( @{$args}{qw/q from size/} ); + push @search, $args->{collapsed} if exists $args->{collapsed}; + my $results = $c->app->model_search->search_web(@search); + + #TODO once output validation works, use this line instead of the one after + #return $c->render(openapi => $results); + return $c->render( json => $results ); +} + +sub first { + +} + +1; + diff --git a/root/static/definitions/parameters.yml b/root/static/definitions/parameters.yml deleted file mode 100644 index 85fee13e6..000000000 --- a/root/static/definitions/parameters.yml +++ /dev/null @@ -1,12 +0,0 @@ ---- - -q: - name: q - description: | - The query search term. If the search term contains a term with the - tags `dist:` or `module:` results will be in expanded form, otherwise - collapsed form. - - See also `collapsed` - type: string - required: true diff --git a/root/static/definitions/results.yml b/root/static/definitions/results.yml index e8c628c8b..95df4cfb5 100644 --- a/root/static/definitions/results.yml +++ b/root/static/definitions/results.yml @@ -5,10 +5,8 @@ search_result_item: properties: description: type: string - description: documentation: type: string - description: authorized: type: boolean path: @@ -40,16 +38,16 @@ search_result_item: type: boolean version_numified: type: number - distribution: - type: string - indexed: - type: boolean, - pod_lines: - type: array - abstract: - type: string - release: - type: string + distribution: + type: string + indexed: + type: boolean + pod_lines: + type: array + abstract: + type: string + release: + type: string dependency: type: object properties: diff --git a/root/static/v1.yml b/root/static/v1.yml index 6f5186cc4..3eeef65eb 100644 --- a/root/static/v1.yml +++ b/root/static/v1.yml @@ -6,11 +6,9 @@ info: title: "MetaCPAN API" basePath: "/v1" tags: - - Search: - name: Search + - name: Search description: MetaCPAN Search Endpoints - - Release: - name: Release + - name: Release description: Distribution Release Endpoints paths: /search/web: @@ -18,17 +16,26 @@ paths: tags: - Search operationId: search_web + x-mojo-to: Search#web summary: Perform API search in the same fashion as the Web UI parameters: - in: query - $ref: "./definitions/parameters.yml#/q" + name: q + description: | + The query search term. If the search term contains a term with the + tags `dist:` or `module:` results will be in expanded form, otherwise + collapsed form. + + See also `collapsed` + type: string + required: true - in: query name: from description: The offset to use in the result set type: integer default: 0 - in: query - name: page_size + name: size description: Number of results per page type: integer default: 20 @@ -47,7 +54,9 @@ paths: total: type: integer took: + type: number collapsed: + type: boolean results: title: Results type: array @@ -60,6 +69,7 @@ paths: tags: - Search operationId: search_for_first + x-mojo-to: Search#first summary: Perform API search and return the first result (I'm Feeling Lucky) parameters: - in: query @@ -135,6 +145,7 @@ paths: tags: - Release operationId: release_recent + x-mojo-to: Release#recent summary: Get recent releases parameters: - in: path @@ -160,6 +171,7 @@ paths: tags: - Release operationId: release_by_name + x-mojo-to: Release#by_name summary: Get details about a release parameters: - in: path From 2f6f0ebf5dcef36dc3d7e3d9b10ca20a717043e9 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 02:22:16 +0000 Subject: [PATCH 068/892] Added Model::User + filled search_identities --- lib/MetaCPAN/API.pm | 6 ++++ lib/MetaCPAN/API/Controller/Admin.pm | 9 ++++- lib/MetaCPAN/Model/User.pm | 42 +++++++++++++++++++++++ t/admin.t | 5 ++- templates/admin/search_identities.html.ep | 2 ++ 5 files changed, 62 insertions(+), 2 deletions(-) create mode 100644 lib/MetaCPAN/Model/User.pm diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index ace32277a..7b67a1744 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -24,6 +24,7 @@ use Config::ZOMG (); use File::Temp (); use List::Util qw( any ); use MetaCPAN::Model::Search (); +use MetaCPAN::Model::User (); use MetaCPAN::Script::Runner (); use Search::Elasticsearch (); use Try::Tiny qw( catch try ); @@ -43,6 +44,11 @@ has model_search => sub { ); }; +has model_user => sub { + my $self = shift; + return MetaCPAN::Model::User->new( es => $self->es, ); +}; + sub startup { my $self = shift; diff --git a/lib/MetaCPAN/API/Controller/Admin.pm b/lib/MetaCPAN/API/Controller/Admin.pm index 6c6a5fb8c..e73937518 100644 --- a/lib/MetaCPAN/API/Controller/Admin.pm +++ b/lib/MetaCPAN/API/Controller/Admin.pm @@ -3,6 +3,13 @@ package MetaCPAN::API::Controller::Admin; use Mojo::Base 'Mojolicious::Controller'; sub identity_search_form { } -sub search_identities { } + +sub search_identities { + my $self = shift; + my $data = $self->app->model_user->lookup( $self->param('name'), + $self->param('key') ); + $self->stash( user_data => $data ); + $self->render('admin/search_identities'); +} 1; diff --git a/lib/MetaCPAN/Model/User.pm b/lib/MetaCPAN/Model/User.pm new file mode 100644 index 000000000..433a93750 --- /dev/null +++ b/lib/MetaCPAN/Model/User.pm @@ -0,0 +1,42 @@ +package MetaCPAN::Model::User; + +use MetaCPAN::Moose; + +use Log::Contextual qw( :log :dlog ); +use MooseX::StrictConstructor; + +use MetaCPAN::Types qw( Object ); + +#use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); + +has es => ( + is => 'ro', + isa => Object, + handles => { _run_query => 'search', }, + required => 1, +); + +sub lookup { + my ( $self, $name, $key ) = @_; + + my $query = { + bool => { + must => [ + { term => { 'identity.name' => $name } }, + { term => { 'identity.key' => $key } }, + ] + } + }; + + my $res = $self->_run_query( + index => 'user', + type => 'account', + body => { query => $query }, + search_type => 'dfs_query_then_fetch', + ); + + return $res->{hits}{hits}[0]{_source}; +} + +1; + diff --git a/t/admin.t b/t/admin.t index b720c8ab4..35e29556e 100644 --- a/t/admin.t +++ b/t/admin.t @@ -30,7 +30,10 @@ subtest 'search identities' => sub { $t->get_ok('/admin/identity-search-form'); $t->status_is(200); - $t->post_ok('/admin/search-identities'); + $t->post_ok( '/admin/search-identities' => form => + { name => 'pause', key => 'MO' } ); + $t->content_like(qr/\bMO\b/); + $t->content_like(qr/\bpause\b/); $t->status_is(200); }; diff --git a/templates/admin/search_identities.html.ep b/templates/admin/search_identities.html.ep index da0175af0..3428abc44 100644 --- a/templates/admin/search_identities.html.ep +++ b/templates/admin/search_identities.html.ep @@ -1 +1,3 @@ display results below: +<%= stash('user_data')->{identity}[0]{name} %> +<%= stash('user_data')->{identity}[0]{key} %> From b805cab51f7f017a60df80939426b46109d8d3c1 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 18:18:44 -0600 Subject: [PATCH 069/892] Upgrade Mojo from 8.05 to 8.06 --- cpanfile.snapshot | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 44b0cfa03..c2bf7a3e0 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4723,8 +4723,8 @@ DISTRIBUTIONS URI::db 0.15 URI::file 4.21 perl 5.010001 - Mojolicious-8.05 - pathname: S/SR/SRI/Mojolicious-8.05.tar.gz + Mojolicious-8.06 + pathname: S/SR/SRI/Mojolicious-8.06.tar.gz provides: Mojo undef Mojo::Asset undef @@ -4794,7 +4794,7 @@ DISTRIBUTIONS Mojo::UserAgent::Transactor undef Mojo::Util undef Mojo::WebSocket undef - Mojolicious 8.05 + Mojolicious 8.06 Mojolicious::Command undef Mojolicious::Command::Author::cpanify undef Mojolicious::Command::Author::generate undef From 064a00132e4658b12863022d86ad231654d720c8 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 18:45:12 -0600 Subject: [PATCH 070/892] Move github credentials and Mojo secret to config file --- lib/MetaCPAN/API.pm | 14 +++++++------- metacpan_server_testing.conf | 5 +++++ 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 7b67a1744..62b437632 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -61,8 +61,9 @@ sub startup { ); } - # TODO secret from config - $self->secrets( [ $ENV{MOJO_SECRET} ] ); + die 'need secret' unless $self->config->{secret}; + + $self->secrets( [ $self->config->{secret} ] ); $self->static->paths( [ $self->home->child('root') ] ); @@ -83,7 +84,7 @@ sub startup { $self->minion->add_task( index_favorite => $self->_gen_index_task_sub('favorite') ); - $self->_maybe_set_up_routes; + $self->_set_up_routes; } sub _gen_index_task_sub { @@ -119,17 +120,16 @@ sub _gen_index_task_sub { } } -sub _maybe_set_up_routes { +sub _set_up_routes { my $self = shift; - return unless $ENV{MOJO_SECRET} && $ENV{GITHUB_KEY}; my $r = $self->routes; $self->plugin( 'Web::Auth', module => 'Github', - key => $ENV{GITHUB_KEY}, - secret => $ENV{GITHUB_SECRET}, + key => $self->config->{github_key}, + secret => $self->config->{github_secret}, on_finished => sub { my ( $c, $access_token, $account_info ) = @_; my $login = $account_info->{login}; diff --git a/metacpan_server_testing.conf b/metacpan_server_testing.conf index 59e200d54..0dbc285e8 100644 --- a/metacpan_server_testing.conf +++ b/metacpan_server_testing.conf @@ -17,3 +17,8 @@ source_base var/t/tmp/source captcha_class Captcha::Mock private_key testing
+ +github_key = foo +github_secret = bar + +secret weak From ed13dd10c5a6529f55c2c14a3ab6f6fd2d6eadfe Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 9 Nov 2018 20:31:42 -0600 Subject: [PATCH 071/892] The ES port is no longer in the config file --- .travis.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 058fc61db..64af31418 100644 --- a/.travis.yml +++ b/.travis.yml @@ -70,7 +70,6 @@ install: - AUTHOR_TESTING=0 cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (tail -n 500 -f ~/.perl-cpm/build.log; false) before_script: - - "perl -i -pe 's/(servers :)9900/localhost:9200/' metacpan_server_testing.conf" - bin/wait-for-open http://localhost:9200/ - coverage-setup From 5859d59390a5adad40e33a3a4a51ab21087159d4 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 10 Nov 2018 16:24:15 +0000 Subject: [PATCH 072/892] ES nodes definition for tests, leave empty for default in production (:9200) --- lib/MetaCPAN/API.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 62b437632..7f22fc787 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -32,7 +32,7 @@ use Try::Tiny qw( catch try ); has es => sub { return Search::Elasticsearch->new( client => '2_0::Direct', - nodes => [':9200'], #TODO config + ( $ENV{ES} ? ( nodes => [ $ENV{ES} ] ) : () ), ); }; From 7f5125d5556e60ff54b7f7ba610020c5fa035c78 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 10 Nov 2018 11:07:54 -0600 Subject: [PATCH 073/892] Change openapi spec to handle new search_web search_web has been updated to provide the v2 results, as that's what's in use by everything. Updating the spec file to match the results that are returned by this path. --- root/static/definitions/results.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/root/static/definitions/results.yml b/root/static/definitions/results.yml index 95df4cfb5..eaf7183f1 100644 --- a/root/static/definitions/results.yml +++ b/root/static/definitions/results.yml @@ -1,6 +1,19 @@ --- -search_result_item: +search_result_items: + type: object + properties: + distribution: + type: string + hits: + title: Hits + type: array + items: + $ref: "#/search_result_hit" + + total: + type: integer +search_result_hit: type: object properties: description: @@ -18,7 +31,9 @@ search_result_item: date: type: string favorites: - type: integer + type: + - "integer" + - "null" status: type: string score: From 3ec7909b77434ce07df9d5a1ad3c49fe02041b45 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 10 Nov 2018 11:09:44 -0600 Subject: [PATCH 074/892] Separate requests from main spec file By splitting out the different request types into separate files the spec is easier to read and makes migrating to the new Mojo routes easier to implement over time. --- root/static/requests/release.yml | 48 +++++++++ root/static/requests/search.yml | 112 +++++++++++++++++++ root/static/v1.yml | 177 +------------------------------ 3 files changed, 161 insertions(+), 176 deletions(-) create mode 100644 root/static/requests/release.yml create mode 100644 root/static/requests/search.yml diff --git a/root/static/requests/release.yml b/root/static/requests/release.yml new file mode 100644 index 000000000..a872d0364 --- /dev/null +++ b/root/static/requests/release.yml @@ -0,0 +1,48 @@ + /release/recent: + get: + tags: + - Release + operationId: release_recent + x-mojo-to: Release#recent + summary: Get recent releases + parameters: + - in: path + name: name + description: | + The name of the Release + type: string + required: true + responses: + 200: + description: Release response + schema: + type: object + properties: + name: + type: string + dependency: + type: array + items: + $ref: "./definitions/results.yml#/dependency" + /release/{name}: + get: + tags: + - Release + operationId: release_by_name + x-mojo-to: Release#by_name + summary: Get details about a release + parameters: + - in: path + name: name + description: | + The name of the Release + type: string + required: true + responses: + 200: + description: Release response + schema: + type: object + properties: + name: + type: string diff --git a/root/static/requests/search.yml b/root/static/requests/search.yml new file mode 100644 index 000000000..7234e3662 --- /dev/null +++ b/root/static/requests/search.yml @@ -0,0 +1,112 @@ +--- + +search_web: + get: + tags: + - Search + operationId: search_web + x-mojo-to: Search#web + summary: Perform API search in the same fashion as the Web UI + parameters: + - name: q + in: query + description: | + The query search term. If the search term contains a term with the + tags `dist:` or `module:` results will be in expanded form, otherwise + collapsed form. + + See also `collapsed` + type: string + required: true + - name: from + in: query + description: The offset to use in the result set + type: integer + default: 0 + - name: size + in: query + description: Number of results per page + type: integer + default: 20 + - name: collapsed + in: query + description: | + Force a collapsed even when searching for a particular + distribution or module name. + type: boolean + responses: + 200: + description: Search response + schema: + type: object + properties: + total: + type: integer + took: + type: number + collapsed: + type: boolean + results: + title: Results + type: array + items: + $ref: "../definitions/results.yml#/search_result_items" +search_first: + get: + tags: + - Search + operationId: search_for_first + x-mojo-to: Search#first + summary: Perform API search and return the first result (I'm Feeling Lucky) + parameters: + - name: q + in: query + description: | + The query search term. + type: string + required: true + responses: + 200: + description: Search response + schema: + type: object + properties: + path: + type: string + description: Relative path to module with full name + authorized: + type: boolean + description: + type: string + description: Module description + id: + type: string + distribution: + type: string + description: Name of the distribution the module is contained in + author: + type: string + description: Module author ID + release: + type: string + description: Package name with version + status: + type: string + abstract.analyzed: + type: string + description: The module's abstract as analyzed from POD + dist_fav_count: + type: integer + description: Number of times favorited + date: + type: string + description: date module was indexed + documentation: + type: string + pod_lines: + type: array + items: + type: integer + indexed: + type: boolean + description: Is the module indexed by PAUSE diff --git a/root/static/v1.yml b/root/static/v1.yml index 3eeef65eb..092a0bc72 100644 --- a/root/static/v1.yml +++ b/root/static/v1.yml @@ -12,179 +12,4 @@ tags: description: Distribution Release Endpoints paths: /search/web: - get: - tags: - - Search - operationId: search_web - x-mojo-to: Search#web - summary: Perform API search in the same fashion as the Web UI - parameters: - - in: query - name: q - description: | - The query search term. If the search term contains a term with the - tags `dist:` or `module:` results will be in expanded form, otherwise - collapsed form. - - See also `collapsed` - type: string - required: true - - in: query - name: from - description: The offset to use in the result set - type: integer - default: 0 - - in: query - name: size - description: Number of results per page - type: integer - default: 20 - - in: query - name: collapsed - description: | - Force a collapsed even when searching for a particular - distribution or module name. - type: boolean - responses: - 200: - description: Search response - schema: - type: object - properties: - total: - type: integer - took: - type: number - collapsed: - type: boolean - results: - title: Results - type: array - items: - type: array - items: - $ref: "./definitions/results.yml#/search_result_item" - /search/first: - get: - tags: - - Search - operationId: search_for_first - x-mojo-to: Search#first - summary: Perform API search and return the first result (I'm Feeling Lucky) - parameters: - - in: query - name: q - description: | - The query search term. - type: string - required: true - responses: - 200: - description: Search response - schema: - type: object - properties: - path: - type: string - description: Relative path to module with full name - # "lib/Moo.pm", - authorized: - type: boolean - # true, - description: - type: string - description: Module description - # "\"Moo\" is an extremely light-weight Object Orientation system. It allows one to concisely define objects and roles with a convenient syntax that avoids the details of Perl's object system. \"Moo\" contains a subset of Moose and is optimised for rapid startup. \"Moo\" avoids depending on any XS modules to allow for simple deployments. The name \"Moo\" is based on the idea that it provides almost -- but not quite -- two thirds of Moose. Unlike Mouse this module does not aim at full compatibility with Moose's surface syntax, preferring instead to provide full interoperability via the metaclass inflation capabilities described in \"MOO AND MOOSE\". For a full list of the minor differences between Moose and Moo's surface syntax, see \"INCOMPATIBILITIES WITH MOOSE\".", - id: - type: string - # "xnl1_tvOXN5leFY9xIUOWrAiRso", - distribution: - type: string - description: Name of the distribution the module is contained in - # "Moo", - author: - type: string - description: Module author ID - # "HAARG", - release: - type: string - description: Package name with version - status: - type: string - # "latest", - abstract.analyzed: - type: string - description: The module's abstract as analyzed from POD - # "Minimalist Object Orientation (with Moose compatibility)", - dist_fav_count: - # $ref: "./definitions/definitions.yml#/dist_fav_count" - type: integer - description: Number of times favorited - # 258, - date: - type: string - description: date module was indexed - # "2017-12-01T01:48:32", - documentation: - type: string - # "Moo", - pod_lines: - type: array - items: - type: integer - # [ - # 254, - # 829 - # ], - indexed: - type: boolean - description: Is the module indexed by PAUSE - # true - /release/recent: - get: - tags: - - Release - operationId: release_recent - x-mojo-to: Release#recent - summary: Get recent releases - parameters: - - in: path - name: name - description: | - The name of the Release - type: string - required: true - responses: - 200: - description: Release response - schema: - type: object - properties: - name: - type: string - dependency: - type: array - items: - $ref: "./definitions/results.yml#/dependency" - /release/{name}: - get: - tags: - - Release - operationId: release_by_name - x-mojo-to: Release#by_name - summary: Get details about a release - parameters: - - in: path - name: name - description: | - The name of the Release - type: string - required: true - responses: - 200: - description: Release response - schema: - type: object - properties: - name: - type: string + $ref: "requests/search.yml#/search_web" From a62731128c219cc55c71ed6a318f7e252f5980ac Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 10 Nov 2018 11:11:34 -0600 Subject: [PATCH 075/892] Add more details to overall spec There are lots of options that can be implemented to provide further documentation of the API. Adding some more now. --- root/static/v1.yml | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/root/static/v1.yml b/root/static/v1.yml index 092a0bc72..9cfb70316 100644 --- a/root/static/v1.yml +++ b/root/static/v1.yml @@ -2,7 +2,7 @@ swagger: "2.0" info: - version: "1.0" + version: "1.0.0" title: "MetaCPAN API" basePath: "/v1" tags: @@ -10,6 +10,13 @@ tags: description: MetaCPAN Search Endpoints - name: Release description: Distribution Release Endpoints +schemes: + - "http" + - "https" +consumes: + - "application/json" +produces: + - "application/json" paths: /search/web: $ref: "requests/search.yml#/search_web" From ca53a54e19a76b7a824bbe744ee6ea2dff4e21fe Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sat, 10 Nov 2018 17:24:55 +0000 Subject: [PATCH 076/892] attach search/first to the mojo router --- lib/MetaCPAN/API/Controller/Search.pm | 20 +++++++++++++------- root/static/v1.yml | 2 ++ 2 files changed, 15 insertions(+), 7 deletions(-) diff --git a/lib/MetaCPAN/API/Controller/Search.pm b/lib/MetaCPAN/API/Controller/Search.pm index bb45970ca..db4b61291 100644 --- a/lib/MetaCPAN/API/Controller/Search.pm +++ b/lib/MetaCPAN/API/Controller/Search.pm @@ -2,6 +2,18 @@ package MetaCPAN::API::Controller::Search; use Mojo::Base 'Mojolicious::Controller'; +has model => sub { shift->app->model_search }; + +sub first { + my $c = shift; + return unless $c->openapi->valid_input; + my $args = $c->validation->output; + + my $results = $c->model->search_for_first_result( $args->{q} ); + return $c->render( openapi => $results ) if $results; + $c->rendered(404); +} + sub web { my $c = shift; return unless $c->openapi->valid_input; @@ -9,16 +21,10 @@ sub web { my @search = ( @{$args}{qw/q from size/} ); push @search, $args->{collapsed} if exists $args->{collapsed}; - my $results = $c->app->model_search->search_web(@search); + my $results = $c->model->search_web(@search); - #TODO once output validation works, use this line instead of the one after - #return $c->render(openapi => $results); return $c->render( json => $results ); } -sub first { - -} - 1; diff --git a/root/static/v1.yml b/root/static/v1.yml index 9cfb70316..b5f4f80af 100644 --- a/root/static/v1.yml +++ b/root/static/v1.yml @@ -20,3 +20,5 @@ produces: paths: /search/web: $ref: "requests/search.yml#/search_web" + /search/first: + $ref: "requests/search.yml#/search_first" From e6939f80773c375e72e96d0323a356a5a49f47b4 Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sat, 10 Nov 2018 18:20:16 +0000 Subject: [PATCH 077/892] add tests for mojo-based search endpoints --- t/api/controller/search/first.t | 16 ++++++++ t/api/controller/search/web.t | 73 +++++++++++++++++++++++++++++++++ 2 files changed, 89 insertions(+) create mode 100644 t/api/controller/search/first.t create mode 100644 t/api/controller/search/web.t diff --git a/t/api/controller/search/first.t b/t/api/controller/search/first.t new file mode 100644 index 000000000..3ca6acdf4 --- /dev/null +++ b/t/api/controller/search/first.t @@ -0,0 +1,16 @@ +use Mojo::Base -strict; + +use Test::More; +use Test::Mojo; +use Mojo::JSON qw(true false); + +my $t = Test::Mojo->new('MetaCPAN::API'); + +$t->get_ok( '/v1/search/first', form => { q => 'Versions::PkgVar' } ) + ->status_is(200)->json_like( '/release' => qr/Versions-(?:\d+)/ ); + +$t->get_ok( '/v1/search/first', form => { q => 'DOESNOTEXISTS' } ) + ->status_is(404)->content_is(''); + +done_testing; + diff --git a/t/api/controller/search/web.t b/t/api/controller/search/web.t new file mode 100644 index 000000000..cc433b980 --- /dev/null +++ b/t/api/controller/search/web.t @@ -0,0 +1,73 @@ +use Mojo::Base -strict; + +use Test::More; +use Test::Mojo; +use Mojo::JSON qw(true false); + +# Note: we need a release marked as status => latest +# so we're using Versions::PkgVar for now +# perhaps it should be smarter later and find one to try? + +my $t = Test::Mojo->new('MetaCPAN::API'); + +subtest 'collapsed' => sub { + $t->get_ok( '/v1/search/web', form => { q => 'Versions::PkgVar' } ) + ->status_is(200)->json_is( '/collapsed' => true ); + + $t->get_ok( '/v1/search/web', form => { q => 'module:Versions::PkgVar' } ) + ->status_is(200)->json_is( '/collapsed' => false ); + + $t->get_ok( '/v1/search/web', + form => { q => 'module:Versions::PkgVar', collapsed => 1 } ) + ->status_is(200)->json_is( '/collapsed' => true ); + + $t->get_ok( '/v1/search/web', form => { q => 'dist:Versions' } ) + ->status_is(200)->json_is( '/collapsed' => false ); + + $t->get_ok( '/v1/search/web', + form => { q => 'dist:Versions', collapsed => 1 } )->status_is(200) + ->json_is( '/collapsed' => true ); + + $t->get_ok( '/v1/search/web', form => { q => 'distribution:Versions' } ) + ->status_is(200)->json_is( '/collapsed' => false ); + + $t->get_ok( '/v1/search/web', + form => { q => 'distribution:Versions', collapsed => 1 } ) + ->status_is(200)->json_is( '/collapsed' => true ); +}; + +subtest 'paging' => sub { + my $q = 'this'; + $t->get_ok( '/v1/search/web', form => { q => $q } )->status_is(200); + my $json = $t->tx->res->json; + my $total = $json->{total}; + + if ( $total <= 1 ) { + cmp_ok @{ $json->{results} }, '==', $total, + 'results agree with total'; + diag "Only one search result, skipping remaining paging tests\n"; + return; + } + + diag "Testing paging with $total results\n"; + cmp_ok @{ $json->{results} }, '<=', $total, 'results agree with total'; + + # shrink the page size to one, test limit + $t->get_ok( '/v1/search/web', form => { q => $q, size => 1 } ) + ->status_is(200)->json_is( '/total' => $total, 'total is unchanged' ); + $json = $t->tx->res->json; + cmp_ok @{ $json->{results} }, '==', 1, 'results has been limited by size'; + my $first = $json->{results}[0]{hits}[0]{id}; + + # keep the page size as one, test offset + $t->get_ok( '/v1/search/web', form => { q => $q, size => 1, from => 1 } ) + ->status_is(200)->json_is( '/total' => $total, 'total is unchanged' ); + $json = $t->tx->res->json; + cmp_ok @{ $json->{results} }, '==', 1, 'results has been limited by size'; + my $next = $json->{results}[0]{hits}[0]{id}; + + isnt $first, $next, 'got a different result'; +}; + +done_testing; + From 2954a2e627cda7f52e075d85dc1ffd9e5f3f501d Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sat, 10 Nov 2018 18:26:50 +0000 Subject: [PATCH 078/892] add symlink from bin/api.pl back to former bin/queue.pl As the queue app grew to become the bigger app that it now is, the name didn't make sense anymore. Thus MetaCPAN::Queue became MetaCPAN::API the script was renamed too. However for ease of migration, this symlink will keep the existing puppet services able to run the minion workers that it originally ran. Once those are updated to point to bin/api.pl this symlink can then be removed. --- bin/queue.pl | 1 + 1 file changed, 1 insertion(+) create mode 120000 bin/queue.pl diff --git a/bin/queue.pl b/bin/queue.pl new file mode 120000 index 000000000..5474dbc6e --- /dev/null +++ b/bin/queue.pl @@ -0,0 +1 @@ +api.pl \ No newline at end of file From 3fb8f1847c9e0a1edf14b62d8570a432c4e062bd Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 10 Nov 2018 16:02:00 -0600 Subject: [PATCH 079/892] Try to capture Mojo logs via AWS --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 64af31418..786af2fe6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,6 +68,7 @@ before_install: install: - AUTHOR_TESTING=0 cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (tail -n 500 -f ~/.perl-cpm/build.log; false) + - mkdir $TRAVIS_BUILD_DIR/log before_script: - bin/wait-for-open http://localhost:9200/ @@ -99,3 +100,4 @@ addons: s3_region: "us-east-1" paths: - $TRAVIS_BUILD_DIR/cpanfile.snapshot + - $TRAVIS_BUILD_DIR/log From 61f50f26e67e9ca07f5887e215283dc6f4e06b09 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 10 Nov 2018 16:31:39 -0600 Subject: [PATCH 080/892] Run prove in verbose mode --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 786af2fe6..d8967edb5 100644 --- a/.travis.yml +++ b/.travis.yml @@ -75,7 +75,7 @@ before_script: - coverage-setup script: - - carton exec prove -It/lib -lr -j$(test-jobs) t + - carton exec prove -It/lib -lrv -j$(test-jobs) t after_success: - coverage-report From 2bc8c81bf7d45fb5c02893db442c8d260d5916c3 Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sat, 10 Nov 2018 23:11:45 +0000 Subject: [PATCH 081/892] move the model loading to a plugin Note, since the models are stateless I'm keeping them in attributes on the plugin itself. This is not the most common pattern, but all it really does is keep instances late-built but cached --- lib/MetaCPAN/API.pm | 16 +------------ lib/MetaCPAN/API/Controller/Admin.pm | 2 +- lib/MetaCPAN/API/Controller/Search.pm | 6 ++--- lib/MetaCPAN/API/Plugin/Model.pm | 34 +++++++++++++++++++++++++++ 4 files changed, 38 insertions(+), 20 deletions(-) create mode 100644 lib/MetaCPAN/API/Plugin/Model.pm diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 7f22fc787..83556707e 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -23,8 +23,6 @@ use Mojo::Base 'Mojolicious'; use Config::ZOMG (); use File::Temp (); use List::Util qw( any ); -use MetaCPAN::Model::Search (); -use MetaCPAN::Model::User (); use MetaCPAN::Script::Runner (); use Search::Elasticsearch (); use Try::Tiny qw( catch try ); @@ -36,19 +34,6 @@ has es => sub { ); }; -has model_search => sub { - my $self = shift; - return MetaCPAN::Model::Search->new( - es => $self->es, - index => 'cpan', - ); -}; - -has model_user => sub { - my $self = shift; - return MetaCPAN::Model::User->new( es => $self->es, ); -}; - sub startup { my $self = shift; @@ -84,6 +69,7 @@ sub startup { $self->minion->add_task( index_favorite => $self->_gen_index_task_sub('favorite') ); + $self->plugin('MetaCPAN::API::Plugin::Model'); $self->_set_up_routes; } diff --git a/lib/MetaCPAN/API/Controller/Admin.pm b/lib/MetaCPAN/API/Controller/Admin.pm index e73937518..a7095cf89 100644 --- a/lib/MetaCPAN/API/Controller/Admin.pm +++ b/lib/MetaCPAN/API/Controller/Admin.pm @@ -6,7 +6,7 @@ sub identity_search_form { } sub search_identities { my $self = shift; - my $data = $self->app->model_user->lookup( $self->param('name'), + my $data = $self->model->user->lookup( $self->param('name'), $self->param('key') ); $self->stash( user_data => $data ); $self->render('admin/search_identities'); diff --git a/lib/MetaCPAN/API/Controller/Search.pm b/lib/MetaCPAN/API/Controller/Search.pm index db4b61291..f0e6ad700 100644 --- a/lib/MetaCPAN/API/Controller/Search.pm +++ b/lib/MetaCPAN/API/Controller/Search.pm @@ -2,14 +2,12 @@ package MetaCPAN::API::Controller::Search; use Mojo::Base 'Mojolicious::Controller'; -has model => sub { shift->app->model_search }; - sub first { my $c = shift; return unless $c->openapi->valid_input; my $args = $c->validation->output; - my $results = $c->model->search_for_first_result( $args->{q} ); + my $results = $c->model->search->search_for_first_result( $args->{q} ); return $c->render( openapi => $results ) if $results; $c->rendered(404); } @@ -21,7 +19,7 @@ sub web { my @search = ( @{$args}{qw/q from size/} ); push @search, $args->{collapsed} if exists $args->{collapsed}; - my $results = $c->model->search_web(@search); + my $results = $c->model->search->search_web(@search); return $c->render( json => $results ); } diff --git a/lib/MetaCPAN/API/Plugin/Model.pm b/lib/MetaCPAN/API/Plugin/Model.pm new file mode 100644 index 000000000..0149ce725 --- /dev/null +++ b/lib/MetaCPAN/API/Plugin/Model.pm @@ -0,0 +1,34 @@ +package MetaCPAN::API::Plugin::Model; + +use Mojo::Base 'Mojolicious::Plugin'; + +use Carp (); +use MetaCPAN::Model::Search (); +use MetaCPAN::Model::User (); + +has app => sub { Carp::croak 'app is required' }, weak => 1; + +has search => sub { + my $self = shift; + return MetaCPAN::Model::Search->new( + es => $self->app->es, + index => 'cpan', + ); +}; + +has user => sub { + my $self = shift; + return MetaCPAN::Model::User->new( es => $self->app->es ); +}; + +sub register { + my ( $plugin, $app, $conf ) = @_; + $plugin->app($app); + + # cached models + $app->helper( 'model.search' => sub { $plugin->search } ); + $app->helper( 'model.user' => sub { $plugin->user } ); +} + +1; + From 80c1370decd8c5fa5b8d721c54c303163652f29a Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sat, 10 Nov 2018 23:18:56 +0000 Subject: [PATCH 082/892] Add the stub of a download model This will hold the find_download_url query as we move it from the Catalyst model which is aparently too bloated to want to save directly. This will be the paradigm for new models. --- lib/MetaCPAN/API/Model/Download.pm | 10 ++++++++++ lib/MetaCPAN/API/Plugin/Model.pm | 17 ++++++++++++++--- 2 files changed, 24 insertions(+), 3 deletions(-) create mode 100644 lib/MetaCPAN/API/Model/Download.pm diff --git a/lib/MetaCPAN/API/Model/Download.pm b/lib/MetaCPAN/API/Model/Download.pm new file mode 100644 index 000000000..32b9fcfa3 --- /dev/null +++ b/lib/MetaCPAN/API/Model/Download.pm @@ -0,0 +1,10 @@ +package MetaCPAN::API::Model::Download; + +use Mojo::Base -base; + +use Carp (); + +has es => sub { Carp::croak 'es is required' }; + +1; + diff --git a/lib/MetaCPAN/API/Plugin/Model.pm b/lib/MetaCPAN/API/Plugin/Model.pm index 0149ce725..99f55cde4 100644 --- a/lib/MetaCPAN/API/Plugin/Model.pm +++ b/lib/MetaCPAN/API/Plugin/Model.pm @@ -2,12 +2,22 @@ package MetaCPAN::API::Plugin::Model; use Mojo::Base 'Mojolicious::Plugin'; -use Carp (); +use Carp (); + +# Models from the catalyst app use MetaCPAN::Model::Search (); use MetaCPAN::Model::User (); +# New models +use MetaCPAN::API::Model::Download (); + has app => sub { Carp::croak 'app is required' }, weak => 1; +has download => sub { + my $self = shift; + return MetaCPAN::API::Model::Download->new( es => $self->app->es ); +}; + has search => sub { my $self = shift; return MetaCPAN::Model::Search->new( @@ -26,8 +36,9 @@ sub register { $plugin->app($app); # cached models - $app->helper( 'model.search' => sub { $plugin->search } ); - $app->helper( 'model.user' => sub { $plugin->user } ); + $app->helper( 'model.download' => sub { $plugin->download } ); + $app->helper( 'model.search' => sub { $plugin->search } ); + $app->helper( 'model.user' => sub { $plugin->user } ); } 1; From 2a055b8e60ee9bcc3836dce53f89133fd5f03c57 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 10 Nov 2018 19:43:19 -0600 Subject: [PATCH 083/892] Add test skips when running on Travis As these tests are essentially wrappers around the t/model/search.t test, and its tests are skipped when running in Travis skip them here. The issue is the same, ES is throwing 'null_pointer_exceptions' when these tests are run, but only in Travis. --- t/api/controller/search/first.t | 4 ++++ t/api/controller/search/web.t | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/t/api/controller/search/first.t b/t/api/controller/search/first.t index 3ca6acdf4..6cfb73942 100644 --- a/t/api/controller/search/first.t +++ b/t/api/controller/search/first.t @@ -4,6 +4,10 @@ use Test::More; use Test::Mojo; use Mojo::JSON qw(true false); +plan skip_all => + "Travis ES bad, see https://travis-ci.org/metacpan/metacpan-api/jobs/301092129" + if $ENV{TRAVIS}; + my $t = Test::Mojo->new('MetaCPAN::API'); $t->get_ok( '/v1/search/first', form => { q => 'Versions::PkgVar' } ) diff --git a/t/api/controller/search/web.t b/t/api/controller/search/web.t index cc433b980..b31aacc8c 100644 --- a/t/api/controller/search/web.t +++ b/t/api/controller/search/web.t @@ -7,6 +7,10 @@ use Mojo::JSON qw(true false); # Note: we need a release marked as status => latest # so we're using Versions::PkgVar for now # perhaps it should be smarter later and find one to try? +# +plan skip_all => + "Travis ES bad, see https://travis-ci.org/metacpan/metacpan-api/jobs/301092129" + if $ENV{TRAVIS}; my $t = Test::Mojo->new('MetaCPAN::API'); From f60925033734957d46021c850b970b2eaacebbf0 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 10 Nov 2018 19:46:35 -0600 Subject: [PATCH 084/892] Remove verbose and Mojo logging These were added while trying to diagnose why Travis builds were failing with the new paths (due to bad ES on Travis was the final reasoning). Removing these are they slow down the tests overall. --- .travis.yml | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/.travis.yml b/.travis.yml index d8967edb5..64af31418 100644 --- a/.travis.yml +++ b/.travis.yml @@ -68,14 +68,13 @@ before_install: install: - AUTHOR_TESTING=0 cpm install -L $PERL_CARTON_PATH --resolver $CPAN_RESOLVER --workers $(test-jobs) || (tail -n 500 -f ~/.perl-cpm/build.log; false) - - mkdir $TRAVIS_BUILD_DIR/log before_script: - bin/wait-for-open http://localhost:9200/ - coverage-setup script: - - carton exec prove -It/lib -lrv -j$(test-jobs) t + - carton exec prove -It/lib -lr -j$(test-jobs) t after_success: - coverage-report @@ -100,4 +99,3 @@ addons: s3_region: "us-east-1" paths: - $TRAVIS_BUILD_DIR/cpanfile.snapshot - - $TRAVIS_BUILD_DIR/log From d90eecbb2a56c65a8759339d1324b51f8680237b Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sun, 11 Nov 2018 16:49:53 +0000 Subject: [PATCH 085/892] move new User model to the new model location at the same time make my new download model stub more like existing models --- lib/MetaCPAN/API/Model/Download.pm | 13 ++++++++++--- lib/MetaCPAN/{ => API}/Model/User.pm | 7 +++---- lib/MetaCPAN/API/Plugin/Model.pm | 4 ++-- 3 files changed, 15 insertions(+), 9 deletions(-) rename lib/MetaCPAN/{ => API}/Model/User.pm (88%) diff --git a/lib/MetaCPAN/API/Model/Download.pm b/lib/MetaCPAN/API/Model/Download.pm index 32b9fcfa3..d871b9943 100644 --- a/lib/MetaCPAN/API/Model/Download.pm +++ b/lib/MetaCPAN/API/Model/Download.pm @@ -1,10 +1,17 @@ package MetaCPAN::API::Model::Download; -use Mojo::Base -base; +use MetaCPAN::Moose; -use Carp (); +use MetaCPAN::Types qw( Object ); -has es => sub { Carp::croak 'es is required' }; +has es => ( + is => 'ro', + isa => Object, + handles => { _run_query => 'search', }, + required => 1, +); + +__PACKAGE__->meta->make_immutable; 1; diff --git a/lib/MetaCPAN/Model/User.pm b/lib/MetaCPAN/API/Model/User.pm similarity index 88% rename from lib/MetaCPAN/Model/User.pm rename to lib/MetaCPAN/API/Model/User.pm index 433a93750..28f356ae6 100644 --- a/lib/MetaCPAN/Model/User.pm +++ b/lib/MetaCPAN/API/Model/User.pm @@ -1,10 +1,7 @@ -package MetaCPAN::Model::User; +package MetaCPAN::API::Model::User; use MetaCPAN::Moose; -use Log::Contextual qw( :log :dlog ); -use MooseX::StrictConstructor; - use MetaCPAN::Types qw( Object ); #use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); @@ -38,5 +35,7 @@ sub lookup { return $res->{hits}{hits}[0]{_source}; } +__PACKAGE__->meta->make_immutable; + 1; diff --git a/lib/MetaCPAN/API/Plugin/Model.pm b/lib/MetaCPAN/API/Plugin/Model.pm index 99f55cde4..157e0230e 100644 --- a/lib/MetaCPAN/API/Plugin/Model.pm +++ b/lib/MetaCPAN/API/Plugin/Model.pm @@ -6,9 +6,9 @@ use Carp (); # Models from the catalyst app use MetaCPAN::Model::Search (); -use MetaCPAN::Model::User (); # New models +use MetaCPAN::API::Model::User (); use MetaCPAN::API::Model::Download (); has app => sub { Carp::croak 'app is required' }, weak => 1; @@ -28,7 +28,7 @@ has search => sub { has user => sub { my $self = shift; - return MetaCPAN::Model::User->new( es => $self->app->es ); + return MetaCPAN::API::Model::User->new( es => $self->app->es ); }; sub register { From efc531aeeebc82f2d14db61595edeb64e261461e Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sun, 11 Nov 2018 11:12:31 -0600 Subject: [PATCH 086/892] Add ReDoc options Update the ReDoc settings with better defaults. --- root/static/index.html | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/root/static/index.html b/root/static/index.html index 00d8388cf..3c119143f 100644 --- a/root/static/index.html +++ b/root/static/index.html @@ -1,7 +1,7 @@ - ReDoc + MetaCPAN API @@ -18,7 +18,7 @@ - + From 3cffcb9305024d6bbbbf176336c0e6493c0683fe Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sun, 11 Nov 2018 11:12:57 -0600 Subject: [PATCH 087/892] Add comments for Release tag Not used at this point in time, commenting them out so that it doesn't show on the ReDoc page. --- root/static/v1.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/root/static/v1.yml b/root/static/v1.yml index b5f4f80af..46e00989d 100644 --- a/root/static/v1.yml +++ b/root/static/v1.yml @@ -8,8 +8,8 @@ basePath: "/v1" tags: - name: Search description: MetaCPAN Search Endpoints - - name: Release - description: Distribution Release Endpoints + # - name: Release + # description: Distribution Release Endpoints schemes: - "http" - "https" From f9425d236a59fd0d489f83e8dfa490efc1075d68 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sun, 11 Nov 2018 17:20:43 +0000 Subject: [PATCH 088/892] MetaCPAN::API::Model - moved the es attribute to a role --- lib/MetaCPAN/API/Model/Download.pm | 9 +-------- lib/MetaCPAN/API/Model/Role/ES.pm | 16 ++++++++++++++++ lib/MetaCPAN/API/Model/User.pm | 11 +---------- 3 files changed, 18 insertions(+), 18 deletions(-) create mode 100644 lib/MetaCPAN/API/Model/Role/ES.pm diff --git a/lib/MetaCPAN/API/Model/Download.pm b/lib/MetaCPAN/API/Model/Download.pm index d871b9943..7900b8bf3 100644 --- a/lib/MetaCPAN/API/Model/Download.pm +++ b/lib/MetaCPAN/API/Model/Download.pm @@ -2,14 +2,7 @@ package MetaCPAN::API::Model::Download; use MetaCPAN::Moose; -use MetaCPAN::Types qw( Object ); - -has es => ( - is => 'ro', - isa => Object, - handles => { _run_query => 'search', }, - required => 1, -); +with 'MetaCPAN::API::Model::Role::ES'; __PACKAGE__->meta->make_immutable; diff --git a/lib/MetaCPAN/API/Model/Role/ES.pm b/lib/MetaCPAN/API/Model/Role/ES.pm new file mode 100644 index 000000000..f6d401989 --- /dev/null +++ b/lib/MetaCPAN/API/Model/Role/ES.pm @@ -0,0 +1,16 @@ +package MetaCPAN::API::Model::Role::ES; + +use Moose::Role; + +use MetaCPAN::Types qw( Object ); + +has es => ( + is => 'ro', + isa => Object, + handles => { _run_query => 'search', }, + required => 1, +); + +no Moose::Role; +1; + diff --git a/lib/MetaCPAN/API/Model/User.pm b/lib/MetaCPAN/API/Model/User.pm index 28f356ae6..81a28c5a8 100644 --- a/lib/MetaCPAN/API/Model/User.pm +++ b/lib/MetaCPAN/API/Model/User.pm @@ -2,16 +2,7 @@ package MetaCPAN::API::Model::User; use MetaCPAN::Moose; -use MetaCPAN::Types qw( Object ); - -#use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); - -has es => ( - is => 'ro', - isa => Object, - handles => { _run_query => 'search', }, - required => 1, -); +with 'MetaCPAN::API::Model::Role::ES'; sub lookup { my ( $self, $name, $key ) = @_; From 34fde32fc5bb375bb9efa71196240b2b2ec2ec08 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sun, 11 Nov 2018 11:42:55 -0600 Subject: [PATCH 089/892] Update documentation for api.pl Now that api.pl is used for all access, update the documentation to reflect it. --- bin/api.pl | 17 +++++++++++++---- lib/MetaCPAN/API.pm | 11 +++++------ 2 files changed, 18 insertions(+), 10 deletions(-) diff --git a/bin/api.pl b/bin/api.pl index 7fd74becd..40ffdc72e 100755 --- a/bin/api.pl +++ b/bin/api.pl @@ -5,25 +5,34 @@ =head2 DESCRIPTION +This is the API web server interface. + + # On vagrant VM + ./bin/run morbo bin/api.pl + +To run the api web server, run the following on one of the servers: + + # Run the daemon on a local port (tunnel to display on your browser) + ./bin/run bin/api.pl daemon + Start Minion worker on vagrant: cd /home/vagrant/metacpan-api - ./bin/run bin/queue.pl minion worker + ./bin/run bin/api.pl minion worker Get status on jobs and workers. On production: - sh /home/metacpan/bin/metacpan-api-carton-exec bin/queue.pl minion job -s + sh /home/metacpan/bin/metacpan-api-carton-exec bin/api.pl minion job -s On vagrant: cd /home/vagrant/metacpan-api - ./bin/run bin/queue.pl minion job -s + ./bin/run bin/api.pl minion job -s =cut -# For vagrant use lib 'lib'; # Start command line interface for application diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 83556707e..e1dec0dc0 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -2,19 +2,18 @@ package MetaCPAN::API; =head1 DESCRIPTION -This is not a web app. It's purely here to manage the API's release indexing -queue. +This is the API web server interface. # On vagrant VM - ./bin/run morbo bin/queue.pl + ./bin/run morbo bin/api.pl # Display information on jobs in queue - ./bin/run bin/queue.pl minion job + ./bin/run bin/api.pl minion job -s -To run the minion admin web interface, run the following on one of the servers: +To run the api web server, run the following on one of the servers: # Run the daemon on a local port (tunnel to display on your browser) - ./bin/run bin/queue.pl daemon + ./bin/run bin/api.pl daemon =cut From bf0cd75487f2004fa4992a65696b1a440c68e067 Mon Sep 17 00:00:00 2001 From: Joel Berger Date: Sun, 11 Nov 2018 18:00:34 +0000 Subject: [PATCH 090/892] move new tests into t/api keeping new modules under MetaCPAN::API and new tests in t/api helps differentiate the new/ported from the old --- t/{ => api/controller}/admin.t | 0 t/{ => api}/queue.t | 0 2 files changed, 0 insertions(+), 0 deletions(-) rename t/{ => api/controller}/admin.t (100%) rename t/{ => api}/queue.t (100%) diff --git a/t/admin.t b/t/api/controller/admin.t similarity index 100% rename from t/admin.t rename to t/api/controller/admin.t diff --git a/t/queue.t b/t/api/queue.t similarity index 100% rename from t/queue.t rename to t/api/queue.t From ddaafac57208c2c93c594a0a1bdb9e303639f244 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sun, 11 Nov 2018 06:05:21 -0600 Subject: [PATCH 091/892] Replace elasticsearch install with docker The Travis-CI implementation of running elasticsearch locally has been throwing null pointer exceptions during testing. Rather than installing a deb package that breaks, run ES inside of a docker container. --- .travis.yml | 10 +++------- 1 file changed, 3 insertions(+), 7 deletions(-) diff --git a/.travis.yml b/.travis.yml index 64af31418..53cd9fa01 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,12 +52,8 @@ before_install: - sudo curl -O -L https://github.com/metacpan/metacpan-puppet/raw/master/modules/metacpan_elasticsearch/files/etc/scripts/score_version_numified.groovy > /tmp/es/score_version_numified.groovy - - sudo cp /tmp/es/* /etc/elasticsearch/scripts/ - - - sudo service elasticsearch stop && curl -O -L https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.4.3.deb && sudo dpkg -i --force-confnew elasticsearch-2.4.3.deb && sudo service elasticsearch start - - - sudo service elasticsearch stop && curl -O -L https://download.elasticsearch.org/elasticsearch/elasticsearch/elasticsearch-2.4.3.deb && sudo dpkg -i --force-confnew elasticsearch-2.4.3.deb && sudo service elasticsearch start - - sudo service elasticsearch restart + - docker pull elasticsearch:2.4-alpine + - docker run -d -p 127.0.0.1:9200:9200 -v /tmp/es:/etc/elasticsearch/scripts elasticsearch:2.4-alpine - cpanm -n Carton - cpanm -n App::cpm @@ -83,7 +79,7 @@ after_success: # - cat ~/.cpanm/build.log services: - - elasticsearch + - docker # caching /local should save about 5 minutes in module install time cache: From 82b0a22683d58b12caf051a1d9d75c87199cf032 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sun, 11 Nov 2018 06:27:50 -0600 Subject: [PATCH 092/892] Fix location of ES scripts The docker image does not look in /etc/elasticsearch/scripts for its scripts. Replacing that with the actual directory. --- .travis.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 53cd9fa01..1a7076627 100644 --- a/.travis.yml +++ b/.travis.yml @@ -53,7 +53,7 @@ before_install: - sudo curl -O -L https://github.com/metacpan/metacpan-puppet/raw/master/modules/metacpan_elasticsearch/files/etc/scripts/score_version_numified.groovy > /tmp/es/score_version_numified.groovy - docker pull elasticsearch:2.4-alpine - - docker run -d -p 127.0.0.1:9200:9200 -v /tmp/es:/etc/elasticsearch/scripts elasticsearch:2.4-alpine + - docker run -d -p 127.0.0.1:9200:9200 -v /tmp/es:/usr/share/elasticsearch/config/scripts elasticsearch:2.4-alpine - cpanm -n Carton - cpanm -n App::cpm From e86a1511b8a9275e6e0dc7213909e940b5afaa45 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sun, 11 Nov 2018 07:09:20 -0600 Subject: [PATCH 093/892] Add ES configuration Using the developer configuration that is used during local testing, with the Travis CI docker testing. --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 1a7076627..c1df5fd4a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,8 +52,10 @@ before_install: - sudo curl -O -L https://github.com/metacpan/metacpan-puppet/raw/master/modules/metacpan_elasticsearch/files/etc/scripts/score_version_numified.groovy > /tmp/es/score_version_numified.groovy + - sudo curl -O -L https://raw.githubusercontent.com/metacpan/metacpan-docker/master/elasticsearch/metacpan.yml > /tmp/metacpan.yml + - docker pull elasticsearch:2.4-alpine - - docker run -d -p 127.0.0.1:9200:9200 -v /tmp/es:/usr/share/elasticsearch/config/scripts elasticsearch:2.4-alpine + - docker run -d -p 127.0.0.1:9200:9200 -v /tmp/metacpan.yml:/usr/share/elasticsearch/config/metacpan.yml -v /tmp/es:/usr/share/elasticsearch/config/scripts elasticsearch:2.4-alpine - cpanm -n Carton - cpanm -n App::cpm From b5b6cd35686d18d0ba601691fbacfb35939a9e5d Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Mon, 19 Nov 2018 14:22:42 -0500 Subject: [PATCH 094/892] Add CHANGES.md to list of possible changes files. (See Perl::Tidy) --- lib/MetaCPAN/Document/File/Set.pm | 2 ++ 1 file changed, 2 insertions(+) diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index 6922b2a51..15bbd1c2a 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -624,6 +624,8 @@ sub find_changes_files { ChangeLog.pod CHANGES Changes + CHANGES.md + CHANGES.markdown CHANGES.pm Changes.pm CHANGES.pod From 3bb6d44ecd5e370c841cfb2a7b8e8989434237dc Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Tue, 20 Nov 2018 21:37:10 -0500 Subject: [PATCH 095/892] Add support for sqlite database with Minion Use the configuration dsn to determine whether Minion should use PostgreSQL or SQLite, allowing developer choice for their database backend. --- lib/MetaCPAN/API.pm | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index e1dec0dc0..f6e184e67 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -51,13 +51,7 @@ sub startup { $self->static->paths( [ $self->home->child('root') ] ); - if ( $ENV{HARNESS_ACTIVE} ) { - my $file = File::Temp->new( UNLINK => 1, SUFFIX => '.db' ); - $self->plugin( Minion => { SQLite => 'sqlite:' . $file } ); - } - else { - $self->plugin( Minion => { Pg => $self->config->{minion_dsn} } ); - } + $self->plugin( Minion => $self->_build_db_params ); $self->minion->add_task( index_release => $self->_gen_index_task_sub('release') ); @@ -171,4 +165,27 @@ sub _is_admin { return any { $username eq $_ } @admins; } +sub _build_db_params { + my $self = shift; + + my $db_params; + if ( $ENV{HARNESS_ACTIVE} ) { + my $file = File::Temp->new( UNLINK => 1, SUFFIX => '.db' ); + return { SQLite => 'sqlite:' . $file }; + } + + die "Unable to determine dsn from configuration" + unless $self->config->{minion_dsn}; + + if ( $self->config->{minion_dsn} =~ /^postgresql:/ ) { + return { Pg => $self->config->{minion_dsn} }; + } + + if ( $self->config->{minion_dsn} =~ /^sqlite:/ ) { + return { SQLite => $self->config->{minion_dsn} }; + } + + die "Unsupported Database in dsn: " . $self->config->{minion_dsn}; +} + 1; From ae595aed8db585b5101054df851e5ece9391baa0 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Sat, 24 Nov 2018 21:47:17 -0500 Subject: [PATCH 096/892] Add route for `/v1` path At some point there will be an update to nginx that no longer strips the `/v1` from the path. When this happens, having this mount point in place will allow the fall through to the Catalyst app to continue. This fixes an issue where the Mojolicous application is unable to validate the specification for OpenAPI because the defined base path does not exist. --- lib/MetaCPAN/API.pm | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index f6e184e67..21ba38098 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -145,6 +145,20 @@ sub _set_up_routes { $self->plugin( 'Minion::Admin' => { route => $admin->any('/minion') } ); $self->plugin( 'OpenAPI' => { url => $self->home->rel_file('root/static/v1.yml') } ); + +# This route is for when nginx gets updated to no longer strip the `/v1` path. +# By retaining the `/v1` path the OpenAPI spec is picked up and passed +# through Mojolicous. The `rewrite` parameter is stripping the `/v1` before +# it is passed to Catalyst allowing the previous logic to be followed. + $self->plugin( + MountPSGI => { + '/v1' => $self->home->child('app.psgi')->to_string, + rewrite => 1 + } + ); + +# XXX Catch cases when `v1` has been stripped by nginx until migration is complete +# XXX then this path can be removed. $self->plugin( MountPSGI => { '/' => $self->home->child('app.psgi')->to_string } ); From 38a3fcff782d450ff154d879822e449fffbae4eb Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 26 Nov 2018 16:13:08 +0100 Subject: [PATCH 097/892] work around PLACK_ENV not being set by MountPSGI --- lib/MetaCPAN/API.pm | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 21ba38098..ed3d1148e 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -26,6 +26,10 @@ use MetaCPAN::Script::Runner (); use Search::Elasticsearch (); use Try::Tiny qw( catch try ); +# MountPSGI doesn't set PLACK_ENV, so Plack::Util::load_psgi defaults to development. +# Hack around that until MountPSGI is fixed. +$ENV{PLACK_ENV} ||= $ENV{'MOJO_MODE'} || 'development'; + has es => sub { return Search::Elasticsearch->new( client => '2_0::Direct', From 19cca503d45de23db429e42ceb885239993fa32b Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 26 Nov 2018 16:20:28 +0100 Subject: [PATCH 098/892] update perltidy --- cpanfile | 2 +- cpanfile.snapshot | 42 +++++++++++----------- lib/Catalyst/Authentication/Store/Proxy.pm | 4 +-- lib/MetaCPAN/API.pm | 2 +- lib/MetaCPAN/Document/File.pm | 6 ++-- lib/MetaCPAN/Document/File/Set.pm | 2 +- lib/MetaCPAN/Model/Release.pm | 2 +- lib/MetaCPAN/Model/Search.pm | 4 +-- lib/MetaCPAN/Script/Author.pm | 4 +-- lib/MetaCPAN/Script/CPANTesters.pm | 2 +- lib/MetaCPAN/Script/CPANTestersAPI.pm | 2 +- lib/MetaCPAN/Script/Pagerank.pm | 2 +- lib/MetaCPAN/Script/Release.pm | 2 +- lib/MetaCPAN/Script/Role/Contributor.pm | 2 +- lib/MetaCPAN/Script/Snapshot.pm | 6 ++-- lib/MetaCPAN/Script/Suggest.pm | 2 +- lib/MetaCPAN/Server/Controller.pm | 2 +- lib/MetaCPAN/Server/Controller/Login.pm | 2 +- lib/MetaCPAN/Server/Model/Source.pm | 2 +- lib/MetaCPAN/Server/View/Pod.pm | 2 +- lib/MetaCPAN/Types/Internal.pm | 2 +- lib/MetaCPAN/Util.pm | 4 +-- t/01_darkpan.t | 2 +- t/lib/Module/Faker/Dist/WithPerl.pm | 2 +- t/model/archive.t | 2 +- t/server/controller/changes.t | 4 +-- 26 files changed, 55 insertions(+), 55 deletions(-) diff --git a/cpanfile b/cpanfile index df6120a0d..14eba3a5c 100644 --- a/cpanfile +++ b/cpanfile @@ -193,7 +193,7 @@ test_requires 'LWP::ConsoleLogger::Easy'; test_requires 'MetaCPAN::Client', '>=', '2.017000'; test_requires 'Module::Faker', '0.015'; test_requires 'Module::Faker::Dist', '0.010'; -test_requires 'Perl::Tidy' => '20180220'; +test_requires 'Perl::Tidy' => '20181120'; test_requires 'Plack::Test::Agent'; test_requires 'Test::Code::TidyAll'; test_requires 'Test::More', '0.99'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index c2bf7a3e0..f41ba58d5 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -7039,27 +7039,27 @@ DISTRIBUTIONS strict 0 version 0.77 warnings 0 - Perl-Tidy-20180220 - pathname: S/SH/SHANCOCK/Perl-Tidy-20180220.tar.gz - provides: - Perl::Tidy 20180220 - Perl::Tidy::Debugger 20180220 - Perl::Tidy::DevNull 20180220 - Perl::Tidy::Diagnostics 20180220 - Perl::Tidy::FileWriter 20180220 - Perl::Tidy::Formatter 20180220 - Perl::Tidy::HtmlWriter 20180220 - Perl::Tidy::IOScalar 20180220 - Perl::Tidy::IOScalarArray 20180220 - Perl::Tidy::IndentationItem 20180220 - Perl::Tidy::LineBuffer 20180220 - Perl::Tidy::LineSink 20180220 - Perl::Tidy::LineSource 20180220 - Perl::Tidy::Logger 20180220 - Perl::Tidy::Tokenizer 20180220 - Perl::Tidy::VerticalAligner 20180220 - Perl::Tidy::VerticalAligner::Alignment 20180220 - Perl::Tidy::VerticalAligner::Line 20180220 + Perl-Tidy-20181120 + pathname: S/SH/SHANCOCK/Perl-Tidy-20181120.tar.gz + provides: + Perl::Tidy 20181120 + Perl::Tidy::Debugger 20181120 + Perl::Tidy::DevNull 20181120 + Perl::Tidy::Diagnostics 20181120 + Perl::Tidy::FileWriter 20181120 + Perl::Tidy::Formatter 20181120 + Perl::Tidy::HtmlWriter 20181120 + Perl::Tidy::IOScalar 20181120 + Perl::Tidy::IOScalarArray 20181120 + Perl::Tidy::IndentationItem 20181120 + Perl::Tidy::LineBuffer 20181120 + Perl::Tidy::LineSink 20181120 + Perl::Tidy::LineSource 20181120 + Perl::Tidy::Logger 20181120 + Perl::Tidy::Tokenizer 20181120 + Perl::Tidy::VerticalAligner 20181120 + Perl::Tidy::VerticalAligner::Alignment 20181120 + Perl::Tidy::VerticalAligner::Line 20181120 requirements: ExtUtils::MakeMaker 0 PerlIO-gzip-0.20 diff --git a/lib/Catalyst/Authentication/Store/Proxy.pm b/lib/Catalyst/Authentication/Store/Proxy.pm index a97673f75..8d445b2a3 100644 --- a/lib/Catalyst/Authentication/Store/Proxy.pm +++ b/lib/Catalyst/Authentication/Store/Proxy.pm @@ -51,7 +51,7 @@ sub new_object { sub from_session { my ( $self, $c, $frozenuser ) = @_; - my $user = $self->new_object( $self->config, $c ); + my $user = $self->new_object( $self->config, $c ); my $delegate = $self->handles->{from_session}; return $user->$delegate( $c, $frozenuser ); } @@ -64,7 +64,7 @@ sub for_session { sub find_user { my ( $self, $authinfo, $c ) = @_; - my $user = $self->new_object( $self->config, $c ); + my $user = $self->new_object( $self->config, $c ); my $delegate = $self->handles->{find_user}; return $user->$delegate( $authinfo, $c ); diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index ed3d1148e..e655ff113 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -100,7 +100,7 @@ sub _gen_index_task_sub { } ); }; - } + } } sub _set_up_routes { diff --git a/lib/MetaCPAN/Document/File.pm b/lib/MetaCPAN/Document/File.pm index 475d6bb51..d2cb8ee1d 100644 --- a/lib/MetaCPAN/Document/File.pm +++ b/lib/MetaCPAN/Document/File.pm @@ -67,7 +67,7 @@ my $RE_SECTION = qr/^\s*(\S+)((\h+-+\h+(.+))|(\r?\n\h*\r?\n\h*(.+)))?/ms; sub _build_section { my $self = shift; - my $text = ${ $self->content }; + my $text = ${ $self->content }; my $section = MetaCPAN::Util::extract_section( $text, 'NAME' ); # if it's a POD file without a name section, let's try to generate @@ -440,7 +440,7 @@ has level => ( ); sub _build_level { - my $self = shift; + my $self = shift; my @level = split( /\//, $self->path ); return @level - 1; } @@ -568,7 +568,7 @@ sub _build_sloc { return 0 unless ( $self->is_perl_file ); my @content = split( "\n", ${ $self->content } ); - my $pods = 0; + my $pods = 0; # Use pod_lines data to remove pod content from string. map { diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index 15bbd1c2a..72bfc7a69 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -372,7 +372,7 @@ sub _version_filters { term => { 'module.version_numified' => $self->_numify($_) } - } + } } @exclusion; } diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 9a0245049..67ccbfad1 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -186,7 +186,7 @@ sub _build_dependencies { sub _build_document { my $self = shift; - my $st = $self->file->stat; + my $st = $self->file->stat; my $stat = { map { $_ => $st->$_ } qw(mode size mtime) }; my $meta = $self->metadata; diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 144c56660..eed257425 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -37,7 +37,7 @@ sub _not_rogue { sub search_for_first_result { my ( $self, $search_term ) = @_; - my $es_query = $self->build_query($search_term); + my $es_query = $self->build_query($search_term); my $es_results = $self->run_query( file => $es_query ); return unless $es_results->{hits}{total}; @@ -381,7 +381,7 @@ sub _extract_results { %{ $res->{fields} }, %{ $res->{_source} }, score => $res->{_score}, - } + } } @{ $es_results->{hits}{hits} } ]; } diff --git a/lib/MetaCPAN/Script/Author.pm b/lib/MetaCPAN/Script/Author.pm index 756fba4d8..e81a69b22 100644 --- a/lib/MetaCPAN/Script/Author.pm +++ b/lib/MetaCPAN/Script/Author.pm @@ -75,14 +75,14 @@ sub index_authors { = ( @$data{qw(fullname email homepage asciiname)} ); $name = undef if ( ref $name ); $asciiname = q{} unless defined $asciiname; - $email = lc($pauseid) . '@cpan.org' + $email = lc($pauseid) . '@cpan.org' unless ( $email && Email::Valid->address($email) ); log_debug { Encode::encode_utf8( sprintf( "Indexing %s: %s <%s>", $pauseid, $name, $email ) ); }; my $conf = $self->author_config( $pauseid, $dates ) || next; - my $put = { + my $put = { pauseid => $pauseid, name => $name, asciiname => ref $asciiname ? undef : $asciiname, diff --git a/lib/MetaCPAN/Script/CPANTesters.pm b/lib/MetaCPAN/Script/CPANTesters.pm index f49120dc5..40199b71b 100644 --- a/lib/MetaCPAN/Script/CPANTesters.pm +++ b/lib/MetaCPAN/Script/CPANTesters.pm @@ -130,7 +130,7 @@ sub index_reports { $version =~ s{\+}{}g; $version =~ s{\A-}{}; - my $release = join( '-', $row_from_db->{dist}, $version ); + my $release = join( '-', $row_from_db->{dist}, $version ); my $release_doc = $releases{$release}; # there's a cpantesters dist we haven't indexed diff --git a/lib/MetaCPAN/Script/CPANTestersAPI.pm b/lib/MetaCPAN/Script/CPANTestersAPI.pm index a2ee0b094..b942793c6 100644 --- a/lib/MetaCPAN/Script/CPANTestersAPI.pm +++ b/lib/MetaCPAN/Script/CPANTestersAPI.pm @@ -95,7 +95,7 @@ sub index_reports { $version =~ s{\+}{}g; $version =~ s{\A-}{}; - my $release = join( '-', $row->{dist}, $version ); + my $release = join( '-', $row->{dist}, $version ); my $release_doc = $releases{$release}; # there's a cpantesters dist we haven't indexed diff --git a/lib/MetaCPAN/Script/Pagerank.pm b/lib/MetaCPAN/Script/Pagerank.pm index 97bb4b74d..9f9553b11 100644 --- a/lib/MetaCPAN/Script/Pagerank.pm +++ b/lib/MetaCPAN/Script/Pagerank.pm @@ -57,7 +57,7 @@ sub run { log_info { "Calculating PageRankg with taking $i dependencies into account"; }; - my $res = $pr->getPagerankOfNodes( listOfEdges => \@edges ); + my $res = $pr->getPagerankOfNodes( listOfEdges => \@edges ); my @sort = sort { $res->{$b} <=> $res->{$a} } keys %$res; for ( 1 .. 500 ) { my $mod = shift @sort; diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index 36310d410..dc5d2fd60 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -222,7 +222,7 @@ sub import_archive { my $self = shift; my $archive_path = shift; - my $bulk = $self->index->bulk( size => $self->_bulk_size ); + my $bulk = $self->index->bulk( size => $self->_bulk_size ); my $model = $self->_get_release_model( $archive_path, $bulk ); log_debug {'Gathering modules'}; diff --git a/lib/MetaCPAN/Script/Role/Contributor.pm b/lib/MetaCPAN/Script/Role/Contributor.pm index 27d3fd500..c8939f159 100644 --- a/lib/MetaCPAN/Script/Role/Contributor.pm +++ b/lib/MetaCPAN/Script/Role/Contributor.pm @@ -21,7 +21,7 @@ sub get_cpan_author_contributors { next unless exists $d->{pauseid}; # skip existing records - my $id = digest( $d->{pauseid}, $release ); + my $id = digest( $d->{pauseid}, $release ); my $exists = $es->exists( index => 'contributor', type => 'contributor', diff --git a/lib/MetaCPAN/Script/Snapshot.pm b/lib/MetaCPAN/Script/Snapshot.pm index 851e66e8e..a6de9c363 100644 --- a/lib/MetaCPAN/Script/Snapshot.pm +++ b/lib/MetaCPAN/Script/Snapshot.pm @@ -16,7 +16,7 @@ use Sys::Hostname qw(hostname); with 'MetaCPAN::Role::Script', 'MooseX::Getopt::Dashes'; my $hostname = hostname; -my $mode = $hostname =~ /dev/ ? 'testing' : 'production'; +my $mode = $hostname =~ /dev/ ? 'testing' : 'production'; # So we dont' break production my $bucket = "mc-${mode}-backups"; @@ -152,7 +152,7 @@ sub run_snapshot { my $snap_name = $self->snap_stub . '_' . $date; my $indices = join ',', @{ $self->indices }; - my $data = { + my $data = { "ignore_unavailable" => 0, "include_global_state" => 1, "indices" => $indices, @@ -170,7 +170,7 @@ sub run_snapshot { sub run_list_snaps { my $self = shift; - my $path = "${repository_name}/_all"; + my $path = "${repository_name}/_all"; my $response = $self->_request( 'get', $path, {} ); my $data = eval { decode_json $response->{content} }; diff --git a/lib/MetaCPAN/Script/Suggest.pm b/lib/MetaCPAN/Script/Suggest.pm index 489872a02..d4065c576 100644 --- a/lib/MetaCPAN/Script/Suggest.pm +++ b/lib/MetaCPAN/Script/Suggest.pm @@ -43,7 +43,7 @@ sub run { log_info {"updating suggest data for month: $gte"}; } - my $lt = $dt->strftime("%Y-%m-%d"); + my $lt = $dt->strftime("%Y-%m-%d"); my $range = +{ range => { date => { gte => $gte, lt => $lt } } }; $self->_update_slice($range); } diff --git a/lib/MetaCPAN/Server/Controller.pm b/lib/MetaCPAN/Server/Controller.pm index 1b3c95777..2d65d1a5d 100644 --- a/lib/MetaCPAN/Server/Controller.pm +++ b/lib/MetaCPAN/Server/Controller.pm @@ -44,7 +44,7 @@ sub apply_request_filter { if ( my $fields = $c->req->param('fields') ) { my $filtered = {}; - my @fields = split /,/, $fields; + my @fields = split /,/, $fields; @$filtered{@fields} = @$data{@fields}; $data = $filtered; } diff --git a/lib/MetaCPAN/Server/Controller/Login.pm b/lib/MetaCPAN/Server/Controller/Login.pm index dafdc1542..aa94770e7 100644 --- a/lib/MetaCPAN/Server/Controller/Login.pm +++ b/lib/MetaCPAN/Server/Controller/Login.pm @@ -38,7 +38,7 @@ sub index : Path { sub update_user { my ( $self, $c, $type, $id, $data ) = @_; my $model = $c->model('User::Account'); - my $user = $model->find( { name => $type, key => $id } ); + my $user = $model->find( { name => $type, key => $id } ); unless ($user) { $user = $model->get( $c->user->id ) if ( $c->session->{__user} ); diff --git a/lib/MetaCPAN/Server/Model/Source.pm b/lib/MetaCPAN/Server/Model/Source.pm index aebfb1940..91aeb6150 100644 --- a/lib/MetaCPAN/Server/Model/Source.pm +++ b/lib/MetaCPAN/Server/Model/Source.pm @@ -54,7 +54,7 @@ sub path { return if -e $source_dir; # previously extracted, but file does not exist my $author = MetaCPAN::Util::author_dir($pauseid); - my $http = dir( qw(var tmp http authors), $author ); + my $http = dir( qw(var tmp http authors), $author ); $author = $self->cpan . "/authors/$author"; my ($archive_file) diff --git a/lib/MetaCPAN/Server/View/Pod.pm b/lib/MetaCPAN/Server/View/Pod.pm index f802fff21..b7d6e57d2 100644 --- a/lib/MetaCPAN/Server/View/Pod.pm +++ b/lib/MetaCPAN/Server/View/Pod.pm @@ -19,7 +19,7 @@ sub process { } my ( $body, $content_type ); - my $accept = eval { $c->req->preferred_content_type } || 'text/html'; + my $accept = eval { $c->req->preferred_content_type } || 'text/html'; my $show_errors = $c->stash->{show_errors}; my $renderer = $self->_factory( diff --git a/lib/MetaCPAN/Types/Internal.pm b/lib/MetaCPAN/Types/Internal.pm index a4b3260a6..9bd9b8cf2 100644 --- a/lib/MetaCPAN/Types/Internal.pm +++ b/lib/MetaCPAN/Types/Internal.pm @@ -121,7 +121,7 @@ subtype RiverSummary, subtype Resources, as Dict [ - license => Optional [ ArrayRef [Str] ], + license => Optional [ ArrayRef [Str] ], homepage => Optional [Str], bugtracker => Optional [ Dict [ web => Optional [Str], mailto => Optional [Str] ] ], diff --git a/lib/MetaCPAN/Util.pm b/lib/MetaCPAN/Util.pm index ce90f82f9..6dad863d8 100644 --- a/lib/MetaCPAN/Util.pm +++ b/lib/MetaCPAN/Util.pm @@ -35,7 +35,7 @@ sub numify_version { $version =~ s/_//g; if ( $version =~ s/^v//i || $version =~ tr/.// > 1 ) { my @parts = split /\./, $version; - my $n = shift @parts; + my $n = shift @parts; return 0 unless defined $n; $version = sprintf( join( '.', '%s', ( '%03s' x @parts ) ), $n, @parts ); @@ -52,7 +52,7 @@ sub fix_version { $version =~ s/\.[._]+/./; $version =~ s/[._]*_[._]*/_/g; $version =~ s/\.{2,}/./g; - $v ||= $version =~ tr/.// > 1; + $v ||= $version =~ tr/.// > 1; $version ||= 0; return ( ( $v ? 'v' : '' ) . $version ); } diff --git a/t/01_darkpan.t b/t/01_darkpan.t index d3f007968..a74c58e13 100644 --- a/t/01_darkpan.t +++ b/t/01_darkpan.t @@ -10,7 +10,7 @@ use Test::More; use Test::RequiresInternet ( 'cpan.metacpan.org' => 80 ); my $darkpan = MetaCPAN::DarkPAN->new; -my $server = MetaCPAN::TestServer->new( cpan_dir => $darkpan->base_dir ); +my $server = MetaCPAN::TestServer->new( cpan_dir => $darkpan->base_dir ); # create DarkPAN $darkpan->run; diff --git a/t/lib/Module/Faker/Dist/WithPerl.pm b/t/lib/Module/Faker/Dist/WithPerl.pm index 0571848c9..8366b5de1 100644 --- a/t/lib/Module/Faker/Dist/WithPerl.pm +++ b/t/lib/Module/Faker/Dist/WithPerl.pm @@ -37,7 +37,7 @@ sub _from_perl_file { my $data = do($filename); my $extra = ( delete $data->{X_Module_Faker} ) || {}; - my $dist = $self->new( { %$data, %$extra } ); + my $dist = $self->new( { %$data, %$extra } ); } __PACKAGE__->meta->make_immutable; diff --git a/t/model/archive.t b/t/model/archive.t index 87dc6f2c2..cb8222c7e 100644 --- a/t/model/archive.t +++ b/t/model/archive.t @@ -14,7 +14,7 @@ subtest 'missing required arguments' => sub { }; subtest 'file does not exist' => sub { - my $file = 'hlaglhalghalghj.blah'; + my $file = 'hlaglhalghalghj.blah'; my $archive = $CLASS->new( file => $file ); throws_ok { $archive->files } qr{$file does not exist}; diff --git a/t/server/controller/changes.t b/t/server/controller/changes.t index 7cdee10a3..9974174c6 100644 --- a/t/server/controller/changes.t +++ b/t/server/controller/changes.t @@ -95,7 +95,7 @@ test_psgi app, sub { for my $test (@tests) { my ( $path, $code, $name, $content, $headers ) = @{$test}; - my $res = get_ok( $cb, $path, $code ); + my $res = get_ok( $cb, $path, $code ); my $json = decode_json_ok($res); test_cache_headers( $res, $headers ); @@ -106,7 +106,7 @@ test_psgi app, sub { like $json->{content}, $content, 'file content'; my @fields = qw(release name content); - $res = get_ok( $cb, "$path?fields=" . join( q[,], @fields ), 200 ); + $res = get_ok( $cb, "$path?fields=" . join( q[,], @fields ), 200 ); $json = decode_json_ok($res); is_deeply [ sort keys %$json ], [ sort @fields ], From 0166e20bfe8067c7f012be367b6e491800370040 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Mon, 26 Nov 2018 16:50:24 -0500 Subject: [PATCH 099/892] Update required version of MountPSGI This version includes a fix that will set the PLACK_ENV variable to the Mojolicious mode. Hopefully turning of developer mode and allowing email to flow, and reduce logging. --- cpanfile | 2 +- cpanfile.snapshot | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/cpanfile b/cpanfile index 14eba3a5c..210910368 100644 --- a/cpanfile +++ b/cpanfile @@ -177,7 +177,7 @@ requires 'strictures', 1; requires 'utf8'; requires 'version', '0.9901'; requires 'warnings'; -requires 'Mojolicious::Plugin::MountPSGI'; +requires 'Mojolicious::Plugin::MountPSGI', '0.14'; requires 'Mojolicious::Plugin::OpenAPI'; requires 'YAML::XS', '0.67'; # Mojolicious::Plugin::OpenAPI YAML loading diff --git a/cpanfile.snapshot b/cpanfile.snapshot index f41ba58d5..17caaef77 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4844,10 +4844,10 @@ DISTRIBUTIONS List::Util 1.41 Time::Local 1.2 perl 5.010001 - Mojolicious-Plugin-MountPSGI-0.13 - pathname: M/MR/MRAMBERG/Mojolicious-Plugin-MountPSGI-0.13.tar.gz + Mojolicious-Plugin-MountPSGI-0.14 + pathname: J/JB/JBERGER/Mojolicious-Plugin-MountPSGI-0.14.tar.gz provides: - Mojolicious::Plugin::MountPSGI 0.13 + Mojolicious::Plugin::MountPSGI 0.14 Mojolicious::Plugin::MountPSGI::Proxy undef requirements: ExtUtils::MakeMaker 0 From 8d2cc6abc2ad3b9ddfdb60dc5b3c80c64ff41b36 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Mon, 26 Nov 2018 16:58:11 -0500 Subject: [PATCH 100/892] Remove setting of PLACK_ENV This should no longer be required as MountPSGI has been updated to set PLACK_ENV. --- lib/MetaCPAN/API.pm | 4 ---- 1 file changed, 4 deletions(-) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index e655ff113..3b073e4de 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -26,10 +26,6 @@ use MetaCPAN::Script::Runner (); use Search::Elasticsearch (); use Try::Tiny qw( catch try ); -# MountPSGI doesn't set PLACK_ENV, so Plack::Util::load_psgi defaults to development. -# Hack around that until MountPSGI is fixed. -$ENV{PLACK_ENV} ||= $ENV{'MOJO_MODE'} || 'development'; - has es => sub { return Search::Elasticsearch->new( client => '2_0::Direct', From 4912ce5623ffebcb6ea397cba189953c491659aa Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Wed, 5 Dec 2018 01:54:39 +0100 Subject: [PATCH 101/892] return full structure even when no results found in author queries --- lib/MetaCPAN/Query/Author.pm | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/lib/MetaCPAN/Query/Author.pm b/lib/MetaCPAN/Query/Author.pm index 609cd8d8a..32b1108f4 100644 --- a/lib/MetaCPAN/Query/Author.pm +++ b/lib/MetaCPAN/Query/Author.pm @@ -26,14 +26,17 @@ sub by_ids { type => 'author', body => $body, ); - return {} unless $authors->{hits}{total}; my @authors = map { single_valued_arrayref_to_scalar( $_->{_source} ); $_->{_source} } @{ $authors->{hits}{hits} }; - return { authors => \@authors }; + return { + authors => \@authors, + took => $authors->{took}, + total => $authors->{hits}{total}, + }; } sub by_user { @@ -48,14 +51,17 @@ sub by_user { size => 500, } ); - return {} unless $authors->{hits}{total}; my @authors = map { single_valued_arrayref_to_scalar( $_->{_source} ); $_->{_source} } @{ $authors->{hits}{hits} }; - return { authors => \@authors }; + return { + authors => \@authors, + took => $authors->{took}, + total => $authors->{hits}{total}, + }; } sub search { @@ -91,7 +97,6 @@ sub search { type => 'author', body => $body, ); - return {} unless $ret->{hits}{total}; my @authors = map { single_valued_arrayref_to_scalar( $_->{_source} ); From 6cd0988ee4a3910b5e310a9ec3028792083fc63b Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Mon, 18 Feb 2019 22:37:48 +0900 Subject: [PATCH 102/892] Add a new API: /changes/by_releases --- lib/MetaCPAN/Document/Release/Set.pm | 1 + lib/MetaCPAN/Query/Release.pm | 45 +++++++++++++++++++++++ lib/MetaCPAN/Server/Controller/Changes.pm | 34 +++++++++++++++++ 3 files changed, 80 insertions(+) diff --git a/lib/MetaCPAN/Document/Release/Set.pm b/lib/MetaCPAN/Document/Release/Set.pm index dbdc58ffd..b22603ec4 100644 --- a/lib/MetaCPAN/Document/Release/Set.pm +++ b/lib/MetaCPAN/Document/Release/Set.pm @@ -20,6 +20,7 @@ has query_release => ( author_status by_author by_author_and_name + by_author_and_names get_contributors get_files latest_by_author diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index 268e71094..a048a166d 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -357,6 +357,51 @@ sub by_author_and_name { }; } +sub by_author_and_names { + my ( $self, $releases ) = @_; + + # $releases: ArrayRef[ Dict[ author => Str, name => Str ] ] + + my $body = { + query => { + bool => { + should => [ + map { +{ + query => { + bool => { + must => [ + { term => { author => uc($_->{author}) } }, + { term => { 'name' => $_->{name} } }, + ] + } + } + } } @$releases + ] + } + } + }; + + my $ret = $self->es->search( + index => $self->index_name, + type => 'release', + body => $body, + ); + return unless $ret->{hits}{total}; + + my @releases; + for my $hit (@{ $ret->{hits}{hits} }) { + my $src = $hit->{_source}; + single_valued_arrayref_to_scalar($src); + push @releases, $src; + } + + return { + took => $ret->{took}, + total => $ret->{hits}{total}, + releases => \@releases, + }; +} + sub by_author { my ( $self, $pauseid, $size ) = @_; $size //= 1000; diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index f83e62a47..8ff20b56c 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -65,4 +65,38 @@ sub all : Chained('index') : PathPart('') : Args(0) { $c->detach('not_found'); } +sub by_releases : Path('by_releases') : Args(0) { + my ( $self, $c ) = @_; + + # ArrayRef[ Dict[ author => Str, name => Str ] ] + my $arg = $c->read_param("releases"); + my $ret = $c->model('CPAN::Release')->by_author_and_names( $arg ); + + my @changes; + for my $release (@{$ret->{releases}}) { + my ($author, $name, $path) = @{$release}{ qw(author name changes_file) }; + my $source = $c->model('Source')->path( $author, $name, $path ) // ''; + + my $content; + try { + local $/; + $content = Encode::decode( + 'UTF-8', + (scalar $source->openr->getline), + Encode::FB_CROAK | Encode::LEAVE_SRC + ); + } catch { + $content = undef; + }; + + push @changes, { + author => $author, + release => $name, + changes_text => $content, + } + } + + $c->stash({ changes => \@changes }); +} + 1; From eec397cd7291bbfb1e66f9c839c5f72a933e753d Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Tue, 19 Feb 2019 08:10:41 +0900 Subject: [PATCH 103/892] tidy --- lib/MetaCPAN/Server/Controller/Changes.pm | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index 8ff20b56c..2c9d25d5f 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -70,11 +70,12 @@ sub by_releases : Path('by_releases') : Args(0) { # ArrayRef[ Dict[ author => Str, name => Str ] ] my $arg = $c->read_param("releases"); - my $ret = $c->model('CPAN::Release')->by_author_and_names( $arg ); + my $ret = $c->model('CPAN::Release')->by_author_and_names($arg); my @changes; - for my $release (@{$ret->{releases}}) { - my ($author, $name, $path) = @{$release}{ qw(author name changes_file) }; + for my $release ( @{ $ret->{releases} } ) { + my ( $author, $name, $path ) + = @{$release}{qw(author name changes_file)}; my $source = $c->model('Source')->path( $author, $name, $path ) // ''; my $content; @@ -82,21 +83,23 @@ sub by_releases : Path('by_releases') : Args(0) { local $/; $content = Encode::decode( 'UTF-8', - (scalar $source->openr->getline), + ( scalar $source->openr->getline ), Encode::FB_CROAK | Encode::LEAVE_SRC ); - } catch { + } + catch { $content = undef; }; - push @changes, { - author => $author, - release => $name, + push @changes, + { + author => $author, + release => $name, changes_text => $content, - } + }; } - $c->stash({ changes => \@changes }); + $c->stash( { changes => \@changes } ); } 1; From 054c20f5794ce5671115dcc23827a30cdb6f9c37 Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Tue, 19 Feb 2019 08:25:07 +0900 Subject: [PATCH 104/892] tidy --- lib/MetaCPAN/Query/Release.pm | 28 +++++++++++++++++----------- 1 file changed, 17 insertions(+), 11 deletions(-) diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index a048a166d..d31682868 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -366,16 +366,22 @@ sub by_author_and_names { query => { bool => { should => [ - map { +{ - query => { - bool => { - must => [ - { term => { author => uc($_->{author}) } }, - { term => { 'name' => $_->{name} } }, - ] + map { + +{ + query => { + bool => { + must => [ + { + term => { + author => uc( $_->{author} ) + } + }, + { term => { 'name' => $_->{name} } }, + ] + } } } - } } @$releases + } @$releases ] } } @@ -389,15 +395,15 @@ sub by_author_and_names { return unless $ret->{hits}{total}; my @releases; - for my $hit (@{ $ret->{hits}{hits} }) { + for my $hit ( @{ $ret->{hits}{hits} } ) { my $src = $hit->{_source}; single_valued_arrayref_to_scalar($src); push @releases, $src; } return { - took => $ret->{took}, - total => $ret->{hits}{total}, + took => $ret->{took}, + total => $ret->{hits}{total}, releases => \@releases, }; } From f6a205ee8ff1dacde25f5822fcc3488eced12187 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Tue, 5 Mar 2019 12:50:54 -0500 Subject: [PATCH 105/892] Change start up call to Mojolicous API metacpan-api is now a Mojolicous app that bootstraps the Catalyst app for routes that are not defined to Mojolicous. As such the command to start the application has changed. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 30275baac..ed2abb738 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ USER metacpan-api:users EXPOSE 5000 -CMD ["carton", "exec", "plackup", "-p", "5000", "-r"] +CMD [ "carton", "exec", "morbo", "-l", "http://*:5000/", "-w", "root", "./bin/api.pl"] From c9cdd81aeb3b9b3f0510e227dc6c55b07227a071 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Tue, 5 Mar 2019 12:52:48 -0500 Subject: [PATCH 106/892] Add wait for the database to be up As there is now a PostgreSQL container that the api container talks to, have to wait for the database container and application to be available before starting the api. The `wait-for-it.sh` script is the accepted way to wait for other services to be available before starting. --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index ed2abb738..e5874307c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -24,4 +24,4 @@ USER metacpan-api:users EXPOSE 5000 -CMD [ "carton", "exec", "morbo", "-l", "http://*:5000/", "-w", "root", "./bin/api.pl"] +CMD [ "./wait-for-it.sh", "db:5432", "carton", "exec", "morbo", "-l", "http://*:5000/", "-w", "root", "./bin/api.pl"] From 90e63f726fc16bdfc1ff60a573cd76bdb1bc2ba2 Mon Sep 17 00:00:00 2001 From: Martin McGrath Date: Wed, 6 Mar 2019 10:16:14 +0000 Subject: [PATCH 107/892] Fix Typo Fix Typo, link to Clinton's slides as intended. --- docs/API-docs.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/API-docs.md b/docs/API-docs.md index b2096a9be..9583cf3cb 100644 --- a/docs/API-docs.md +++ b/docs/API-docs.md @@ -6,7 +6,7 @@ There is also [a repository of examples](https://github.com/metacpan/metacpan-ex _All of these URLs can be tested using the [MetaCPAN Explorer](https://explorer.metacpan.org)_ -To learn more about the ElasticSearch query DSL (Domain-Specific Language) check out Clinton Gormley's [Terms of Endearment - ES Query DSL Explained] (https://www.slideshare.net/clintongormley/terms-of-endearment-the-elasticsearch-query-dsl-explained) slides. +To learn more about the ElasticSearch query DSL (Domain-Specific Language) check out Clinton Gormley's [Terms of Endearment - ES Query DSL Explained](https://www.slideshare.net/clintongormley/terms-of-endearment-the-elasticsearch-query-dsl-explained) slides. The query syntax is explained on ElasticSearch's [reference page](https://www.elasticsearch.org/guide/reference/query-dsl/). You can also check out this getting started tutorial about Elasticsearch [reference page](http://joelabrahamsson.com/elasticsearch-101/). From f51769c5ff859cfd25060d99c9ef105a65b7efa9 Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Fri, 8 Mar 2019 09:19:57 +0900 Subject: [PATCH 108/892] skip the iteration when changelog cannot be read properly. --- lib/MetaCPAN/Server/Controller/Changes.pm | 15 +++++---------- 1 file changed, 5 insertions(+), 10 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index 2c9d25d5f..9291bf65d 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -76,20 +76,15 @@ sub by_releases : Path('by_releases') : Args(0) { for my $release ( @{ $ret->{releases} } ) { my ( $author, $name, $path ) = @{$release}{qw(author name changes_file)}; - my $source = $c->model('Source')->path( $author, $name, $path ) // ''; + my $source = $c->model('Source')->path( $author, $name, $path ) or next; - my $content; - try { - local $/; - $content = Encode::decode( + my $content = try { + Encode::decode( 'UTF-8', - ( scalar $source->openr->getline ), + ( scalar $source->slurp ), Encode::FB_CROAK | Encode::LEAVE_SRC ); - } - catch { - $content = undef; - }; + } or next; push @changes, { From 8e675d230c1a499ac8a8bf178aedff125acf9f61 Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Fri, 8 Mar 2019 09:21:37 +0900 Subject: [PATCH 109/892] include the name of chaneglog in the response. --- lib/MetaCPAN/Server/Controller/Changes.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index 9291bf65d..1b67cd1c3 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -91,6 +91,7 @@ sub by_releases : Path('by_releases') : Args(0) { author => $author, release => $name, changes_text => $content, + changes_file => $path, }; } From 9b27d3ca5cddea49094ff9a6f839afab99671d87 Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Fri, 8 Mar 2019 09:31:14 +0900 Subject: [PATCH 110/892] tidy --- lib/MetaCPAN/Server/Controller/Changes.pm | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index 1b67cd1c3..7b59e5f32 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -76,7 +76,8 @@ sub by_releases : Path('by_releases') : Args(0) { for my $release ( @{ $ret->{releases} } ) { my ( $author, $name, $path ) = @{$release}{qw(author name changes_file)}; - my $source = $c->model('Source')->path( $author, $name, $path ) or next; + my $source = $c->model('Source')->path( $author, $name, $path ) + or next; my $content = try { Encode::decode( From 1da8c5d8b44125c260110b3f07cb8f0bc114da7c Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Tue, 19 Jun 2018 17:10:24 +0200 Subject: [PATCH 111/892] Log::Any via Log::Log4perl Log::Any is used by Search::Elasticsearch, so we should connect it to our standard logger. --- cpanfile | 2 ++ cpanfile.snapshot | 13 +++++++++++++ lib/MetaCPAN/Model.pm | 3 +++ 3 files changed, 18 insertions(+) diff --git a/cpanfile b/cpanfile index 210910368..0f5fda4e7 100644 --- a/cpanfile +++ b/cpanfile @@ -87,6 +87,8 @@ requires 'LWP::UserAgent::Paranoid'; requires 'List::AllUtils', '0.09'; requires 'List::MoreUtils', '0.413'; requires 'List::Util', '1.45'; +requires 'Log::Any::Adapter'; +requires 'Log::Any::Adapter::Log4perl'; requires 'Log::Contextual'; requires 'Log::Log4perl'; requires 'Log::Log4perl::Appender::ScreenColoredLevels'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 17caaef77..3caa5374d 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4061,6 +4061,19 @@ DISTRIBUTIONS constant 0 strict 0 warnings 0 + Log-Any-Adapter-Log4perl-0.09 + pathname: P/PR/PREACTION/Log-Any-Adapter-Log4perl-0.09.tar.gz + provides: + Log::Any::Adapter::Log4perl 0.09 + requirements: + ExtUtils::MakeMaker 6.17 + Log::Any::Adapter::Base 0 + Log::Any::Adapter::Util 1.03 + Log::Log4perl 1.32 + base 0 + perl 5.006 + strict 0 + warnings 0 Log-Contextual-0.007001 pathname: F/FR/FREW/Log-Contextual-0.007001.tar.gz provides: diff --git a/lib/MetaCPAN/Model.pm b/lib/MetaCPAN/Model.pm index 782045640..fc96fe440 100644 --- a/lib/MetaCPAN/Model.pm +++ b/lib/MetaCPAN/Model.pm @@ -4,6 +4,9 @@ package MetaCPAN::Model; use Moose; use ElasticSearchX::Model; +use Log::Any::Adapter; + +Log::Any::Adapter->set('Log4perl'); analyzer lowercase => ( tokenizer => 'keyword', From 8f525624e39fa9f0a98aec9e390e78c967ad13df Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Sat, 9 Mar 2019 14:50:12 +0900 Subject: [PATCH 112/892] adjust the form of input to be conventional. As @haarg++ pointed out, this makes it poosible to retrieve with GET method like: curl 'http://localhost:5000/changes/by_releases?release=YAPPO/LINE-Bot-API-0.04&release=GUGOD/Hijk-0.28' --- lib/MetaCPAN/Server/Controller/Changes.pm | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index 7b59e5f32..897c0459d 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -68,9 +68,16 @@ sub all : Chained('index') : PathPart('') : Args(0) { sub by_releases : Path('by_releases') : Args(0) { my ( $self, $c ) = @_; - # ArrayRef[ Dict[ author => Str, name => Str ] ] - my $arg = $c->read_param("releases"); - my $ret = $c->model('CPAN::Release')->by_author_and_names($arg); + my $ret = $c->model('CPAN::Release')->by_author_and_names([ + map { + my @o = split('/', $_, 2); + @o != 2 ? () : (+{ + author => $o[0], + name => $o[1], + }) + } + @{ $c->read_param("release") } + ]); my @changes; for my $release ( @{ $ret->{releases} } ) { From b7bad82d26e03575aedb9fe765e566e9a9e057b7 Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Sat, 9 Mar 2019 15:07:16 +0900 Subject: [PATCH 113/892] ensure that the retrieved response is always within range of of input. When some of the 'release' parameter is not recognized as the format of "AUTHOR/DistName-v1.23", it shall be simply ignored as if that parameter simply does not exists. Also, avoid constructing an empty ES query -- which would recall everything. --- lib/MetaCPAN/Server/Controller/Changes.pm | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) diff --git a/lib/MetaCPAN/Server/Controller/Changes.pm b/lib/MetaCPAN/Server/Controller/Changes.pm index 897c0459d..6f0294f2b 100644 --- a/lib/MetaCPAN/Server/Controller/Changes.pm +++ b/lib/MetaCPAN/Server/Controller/Changes.pm @@ -68,16 +68,17 @@ sub all : Chained('index') : PathPart('') : Args(0) { sub by_releases : Path('by_releases') : Args(0) { my ( $self, $c ) = @_; - my $ret = $c->model('CPAN::Release')->by_author_and_names([ - map { - my @o = split('/', $_, 2); - @o != 2 ? () : (+{ - author => $o[0], - name => $o[1], - }) - } - @{ $c->read_param("release") } - ]); + my @releases = map { + my @o = split( '/', $_, 2 ); + @o == 2 ? { author => $o[0], name => $o[1] } : (); + } @{ $c->read_param("release") }; + + unless (@releases) { + $c->stash( { changes => [] } ); + return; + } + + my $ret = $c->model('CPAN::Release')->by_author_and_names( \@releases ); my @changes; for my $release ( @{ $ret->{releases} } ) { From 3e6c948df89cf0d97882243e41fe44e78c5afdde Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Wed, 20 Mar 2019 16:49:53 -0400 Subject: [PATCH 114/892] Add wait-for-it.sh script to container The script to wait for the database server to be up needs to be included as part of the container in order for it to be executed. This change adds the script and copies it into position within the container. --- Dockerfile | 3 +- wait-for-it.sh | 202 +++++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 204 insertions(+), 1 deletion(-) create mode 100755 wait-for-it.sh diff --git a/Dockerfile b/Dockerfile index e5874307c..a7c421c62 100644 --- a/Dockerfile +++ b/Dockerfile @@ -2,6 +2,7 @@ FROM perl:5.22 ENV PERL_MM_USE_DEFAULT=1 PERL_CARTON_PATH=/carton +COPY wait-for-it.sh / COPY cpanfile cpanfile.snapshot /metacpan-api/ WORKDIR /metacpan-api @@ -24,4 +25,4 @@ USER metacpan-api:users EXPOSE 5000 -CMD [ "./wait-for-it.sh", "db:5432", "carton", "exec", "morbo", "-l", "http://*:5000/", "-w", "root", "./bin/api.pl"] +CMD [ "/wait-for-it.sh", "db:5432", "--", "carton", "exec", "morbo", "-l", "http://*:5000/", "-w", "root", "./bin/api.pl"] diff --git a/wait-for-it.sh b/wait-for-it.sh new file mode 100755 index 000000000..33e0d0e00 --- /dev/null +++ b/wait-for-it.sh @@ -0,0 +1,202 @@ +#!/bin/bash +# +# See https://github.com/vishnubob/wait-for-it +# +# The MIT License (MIT) +# Copyright (c) 2016 Giles Hall + +# Permission is hereby granted, free of charge, to any person obtaining a copy of +# this software and associated documentation files (the "Software"), to deal in +# the Software without restriction, including without limitation the rights to +# use, copy, modify, merge, publish, distribute, sublicense, and/or sell copies +# of the Software, and to permit persons to whom the Software is furnished to do +# so, subject to the following conditions: + +# The above copyright notice and this permission notice shall be included in all +# copies or substantial portions of the Software. + +# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +# IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +# AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +# LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +# SOFTWARE. + +#!/usr/bin/env bash +# Use this script to test if a given TCP host/port are available + +cmdname=$(basename $0) + +echoerr() { if [[ $QUIET -ne 1 ]]; then echo "$@" 1>&2; fi } + +usage() +{ + cat << USAGE >&2 +Usage: + $cmdname host:port [-s] [-t timeout] [-- command args] + -h HOST | --host=HOST Host or IP under test + -p PORT | --port=PORT TCP port under test + Alternatively, you specify the host and port as host:port + -s | --strict Only execute subcommand if the test succeeds + -q | --quiet Don't output any status messages + -t TIMEOUT | --timeout=TIMEOUT + Timeout in seconds, zero for no timeout + -- COMMAND ARGS Execute command with args after the test finishes +USAGE + exit 1 +} + +wait_for() +{ + if [[ $TIMEOUT -gt 0 ]]; then + echoerr "$cmdname: waiting $TIMEOUT seconds for $HOST:$PORT" + else + echoerr "$cmdname: waiting for $HOST:$PORT without a timeout" + fi + start_ts=$(date +%s) + while : + do + if [[ $ISBUSY -eq 1 ]]; then + nc -z $HOST $PORT + result=$? + else + (echo > /dev/tcp/$HOST/$PORT) >/dev/null 2>&1 + result=$? + fi + if [[ $result -eq 0 ]]; then + end_ts=$(date +%s) + echoerr "$cmdname: $HOST:$PORT is available after $((end_ts - start_ts)) seconds" + break + fi + sleep 1 + done + return $result +} + +wait_for_wrapper() +{ + # In order to support SIGINT during timeout: http://unix.stackexchange.com/a/57692 + if [[ $QUIET -eq 1 ]]; then + timeout $BUSYTIMEFLAG $TIMEOUT $0 --quiet --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + else + timeout $BUSYTIMEFLAG $TIMEOUT $0 --child --host=$HOST --port=$PORT --timeout=$TIMEOUT & + fi + PID=$! + trap "kill -INT -$PID" INT + wait $PID + RESULT=$? + if [[ $RESULT -ne 0 ]]; then + echoerr "$cmdname: timeout occurred after waiting $TIMEOUT seconds for $HOST:$PORT" + fi + return $RESULT +} + +# process arguments +while [[ $# -gt 0 ]] +do + case "$1" in + *:* ) + hostport=(${1//:/ }) + HOST=${hostport[0]} + PORT=${hostport[1]} + shift 1 + ;; + --child) + CHILD=1 + shift 1 + ;; + -q | --quiet) + QUIET=1 + shift 1 + ;; + -s | --strict) + STRICT=1 + shift 1 + ;; + -h) + HOST="$2" + if [[ $HOST == "" ]]; then break; fi + shift 2 + ;; + --host=*) + HOST="${1#*=}" + shift 1 + ;; + -p) + PORT="$2" + if [[ $PORT == "" ]]; then break; fi + shift 2 + ;; + --port=*) + PORT="${1#*=}" + shift 1 + ;; + -t) + TIMEOUT="$2" + if [[ $TIMEOUT == "" ]]; then break; fi + shift 2 + ;; + --timeout=*) + TIMEOUT="${1#*=}" + shift 1 + ;; + --) + shift + CLI=("$@") + break + ;; + --help) + usage + ;; + *) + echoerr "Unknown argument: $1" + usage + ;; + esac +done + +if [[ "$HOST" == "" || "$PORT" == "" ]]; then + echoerr "Error: you need to provide a host and port to test." + usage +fi + +TIMEOUT=${TIMEOUT:-15} +STRICT=${STRICT:-0} +CHILD=${CHILD:-0} +QUIET=${QUIET:-0} + +# check to see if timeout is from busybox? +# check to see if timeout is from busybox? +TIMEOUT_PATH=$(realpath $(which timeout)) +if [[ $TIMEOUT_PATH =~ "busybox" ]]; then + ISBUSY=1 + BUSYTIMEFLAG="-t" +else + ISBUSY=0 + BUSYTIMEFLAG="" +fi + +if [[ $CHILD -gt 0 ]]; then + wait_for + RESULT=$? + exit $RESULT +else + if [[ $TIMEOUT -gt 0 ]]; then + wait_for_wrapper + RESULT=$? + else + wait_for + RESULT=$? + fi +fi + +if [[ $CLI != "" ]]; then + if [[ $RESULT -ne 0 && $STRICT -eq 1 ]]; then + echoerr "$cmdname: strict mode, refusing to execute subprocess" + exit $RESULT + fi + exec "${CLI[@]}" +else + exit $RESULT +fi From 65e49a9e47802fd42edf144c8a78fa7f7840ccad Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 11 Apr 2019 17:52:36 -0400 Subject: [PATCH 115/892] Move PAUSE email sending into a model class --- lib/MetaCPAN/Model/Email/PAUSE.pm | 88 +++++++++++++++++++ lib/MetaCPAN/Server/Controller/Login/PAUSE.pm | 60 ++++--------- lib/MetaCPAN/Util.pm | 6 +- t/model/email/pause.t | 50 +++++++++++ t/util.t | 12 ++- 5 files changed, 167 insertions(+), 49 deletions(-) create mode 100644 lib/MetaCPAN/Model/Email/PAUSE.pm create mode 100644 t/model/email/pause.t diff --git a/lib/MetaCPAN/Model/Email/PAUSE.pm b/lib/MetaCPAN/Model/Email/PAUSE.pm new file mode 100644 index 000000000..e4968e064 --- /dev/null +++ b/lib/MetaCPAN/Model/Email/PAUSE.pm @@ -0,0 +1,88 @@ +package MetaCPAN::Model::Email::PAUSE; + +use MetaCPAN::Moose; + +use Email::Sender::Simple qw( sendmail ); +use Email::Sender::Transport::SMTP (); +use Email::Simple (); +use Encode (); +use MetaCPAN::Types qw( Object Uri ); +use Try::Tiny qw( catch try ); + +has _author => ( + is => 'ro', + isa => Object, + init_arg => 'author', + required => 1, +); + +has _url => ( + is => 'ro', + isa => Uri, + init_arg => 'url', + required => 1, +); + +sub send { + my $self = shift; + + my $email = Email::Simple->create( + header => [ + 'Content-Type' => 'text/plain; charset=utf-8', + To => $self->_author->{email}->[0], + From => 'noreply@metacpan.org', + Subject => 'Connect MetaCPAN with your PAUSE account', + 'MIME-Version' => '1.0', + ], + body => $self->email_body, + ); + + my $transport = Email::Sender::Transport::SMTP->new( + { + host => 'smtp.fastmail.com', + port => 465, + sasl_username => 'foo', + sasl_password => 'bar', + } + ); + + my $success = 0; + try { + sendmail( $email, { transport => $transport } ); + $success = 1; + } + catch { + warn $_; + }; + + return $success; +} + +sub email_body { + my $self = shift; + my $name = $self->_author->name; + my $uri = $self->_url; + + my $body = < ( is => 'ro', + isa => 'CHI::Driver', builder => '_build_cache', ); @@ -44,48 +42,22 @@ sub index : Path { my $author = $c->model('CPAN::Author')->get( uc($id) ); $c->controller('OAuth2')->redirect( $c, error => "author_not_found" ) unless ($author); - my $code = MetaCPAN::Util::generate_sid; - $self->cache->set( $code, $author->pauseid, 86400 ); - my $uri = $c->request->uri->clone; - $uri->query("code=$code"); - my $email = Email::Simple->create( - header => [ - 'Content-Type' => 'text/plain; charset=utf-8', - To => $author->{email}->[0], - From => 'noreply@metacpan.org', - Subject => "Connect MetaCPAN with your PAUSE account", - 'MIME-Version' => '1.0', - ], - body => $self->email_body( $author->name, $uri ), - ); - Email::Sender::Simple->send($email); - $c->controller('OAuth2')->redirect( $c, success => "mail_sent" ); - } -} - -sub email_body { - my ( $self, $name, $uri ) = @_; - my $body = <cache->set( $code, $author->pauseid, 86400 ); -$uri + my $url = $c->request->uri->clone; + $url->query("code=$code"); + my $email = MetaCPAN::Model::Email::PAUSE->new( + author => $author, + url => $url, + ); -Cheers, -MetaCPAN -EMAIL_BODY + my $sent = $email->send; - try { - $body = Encode::encode( 'UTF-8', $body, - Encode::FB_CROAK | Encode::LEAVE_SRC ); + # XXX check return value of sending + $c->controller('OAuth2')->redirect( $c, success => 'mail_sent' ); } - catch { - warn $_[0]; - }; - - return $body; } 1; diff --git a/lib/MetaCPAN/Util.pm b/lib/MetaCPAN/Util.pm index 6dad863d8..4deec300a 100644 --- a/lib/MetaCPAN/Util.pm +++ b/lib/MetaCPAN/Util.pm @@ -13,9 +13,9 @@ use Sub::Exporter -setup => { exports => [ 'author_dir', 'digest', 'extract_section', 'fix_pod', - 'fix_version', 'numify_version', - 'pod_lines', 'strip_pod', - 'single_valued_arrayref_to_scalar' + 'fix_version', 'generate_sid', + 'numify_version', 'pod_lines', + 'strip_pod', 'single_valued_arrayref_to_scalar' ] }; diff --git a/t/model/email/pause.t b/t/model/email/pause.t new file mode 100644 index 000000000..42e93c4c4 --- /dev/null +++ b/t/model/email/pause.t @@ -0,0 +1,50 @@ +use strict; +use warnings; + +## no critic (Modules::RequireFilenameMatchesPackage) +package Author; + +use MetaCPAN::Moose; + +use MetaCPAN::Types qw( ArrayRef Str ); + +has name => ( + is => 'ro', + isa => Str, + init_arg => 'name', +); + +has email => ( + is => 'ro', + isa => ArrayRef [Str], + required => 1, +); + +__PACKAGE__->meta->make_immutable; +1; + +package main; + +BEGIN { $ENV{EMAIL_SENDER_TRANSPORT} = 'Test' } + +use Test::More; + +use MetaCPAN::Model::Email::PAUSE (); + +my $author = Author->new( + name => 'Olaf Alders', + email => ['oalders@metacpan.org'], +); + +my $email = MetaCPAN::Model::Email::PAUSE->new( + author => $author, + url => URI->new('http://example.com/'), +); + +ok( $email->send, 'send email' ); + +my @messages = Email::Sender::Simple->default_transport->deliveries; +is( @messages, 1, '1 message sent' ); + +done_testing(); +1; diff --git a/t/util.t b/t/util.t index f99ad4e85..365495def 100644 --- a/t/util.t +++ b/t/util.t @@ -2,10 +2,18 @@ use strict; use warnings; use lib 't/lib'; -use CPAN::Meta; -use MetaCPAN::Util qw( extract_section numify_version strip_pod ); +use CPAN::Meta (); +use MetaCPAN::Util qw( + extract_section + generate_sid + numify_version + strip_pod +); + use Test::Most; +ok( generate_sid(), 'generate_sid' ); + { my %versions = ( '010' => 10, From db48acf8ede25cd0138f97909dd3120f5510aa04 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 11 Apr 2019 21:32:20 -0400 Subject: [PATCH 116/892] Use more explicit imports in MetaCPAN::Util --- lib/MetaCPAN/Util.pm | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/MetaCPAN/Util.pm b/lib/MetaCPAN/Util.pm index 4deec300a..7fb4a37fd 100644 --- a/lib/MetaCPAN/Util.pm +++ b/lib/MetaCPAN/Util.pm @@ -6,8 +6,8 @@ use strict; use warnings; use version; -use Digest::SHA; -use Encode; +use Digest::SHA qw( sha1_base64 sha1_hex ); +use Encode qw( decode_utf8 ); use Ref::Util qw( is_arrayref is_hashref ); use Sub::Exporter -setup => { exports => [ @@ -20,13 +20,13 @@ use Sub::Exporter -setup => { }; sub digest { - my $digest = Digest::SHA::sha1_base64( join( "\0", grep {defined} @_ ) ); + my $digest = sha1_base64( join( "\0", grep {defined} @_ ) ); $digest =~ tr/[+\/]/-_/; return $digest; } sub generate_sid { - Digest::SHA::sha1_hex( rand() . $$ . {} . time ); + return sha1_hex( rand . $$ . {} . time ); } sub numify_version { @@ -76,7 +76,7 @@ sub strip_pod { sub extract_section { my ( $pod, $section ) = @_; - eval { $pod = Encode::decode_utf8( $pod, Encode::FB_CROAK ) }; + eval { $pod = decode_utf8( $pod, Encode::FB_CROAK ) }; return undef unless ( $pod =~ /^=head1\s+$section\b(.*?)(^((\=head1)|(\=cut)))/msi || $pod =~ /^=head1\s+$section\b(.*)/msi ); From 4e99606f836c57d2364571abd8691d525e32c108 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 12 Apr 2019 11:56:59 -0400 Subject: [PATCH 117/892] Add Authen::SASL and MIME::Base64 to cpanfile --- cpanfile | 2 ++ cpanfile.snapshot | 24 ++++++++++++++++++++++++ 2 files changed, 26 insertions(+) diff --git a/cpanfile b/cpanfile index 0f5fda4e7..01c0d43d4 100644 --- a/cpanfile +++ b/cpanfile @@ -2,6 +2,7 @@ requires 'perl', '5.010'; requires 'Archive::Any', 0.0942; requires 'Archive::Tar', '2.04'; +requires 'Authen::SASL', '2.16'; requires 'BackPAN::Index', '0.42'; requires 'CHI', '0.60'; requires 'CPAN::DistnameInfo', '0.12'; @@ -95,6 +96,7 @@ requires 'Log::Log4perl::Appender::ScreenColoredLevels'; requires 'MetaCPAN::Moose'; requires 'MetaCPAN::Pod::XHTML'; requires 'MetaCPAN::Role', '0.06'; +requires 'MIME::Base64', '3.15'; requires 'Minion', '>= 9.03'; requires 'Minion::Backend::SQLite'; requires 'Module::Load'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 3caa5374d..a5da179ed 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -198,6 +198,27 @@ DISTRIBUTIONS Array::Iterator::Reusable 0.11 requirements: ExtUtils::MakeMaker 6.30 + Authen-SASL-2.16 + pathname: G/GB/GBARR/Authen-SASL-2.16.tar.gz + provides: + Authen::SASL 2.16 + Authen::SASL::CRAM_MD5 2.14 + Authen::SASL::EXTERNAL 2.14 + Authen::SASL::Perl 2.14 + Authen::SASL::Perl::ANONYMOUS 2.14 + Authen::SASL::Perl::CRAM_MD5 2.14 + Authen::SASL::Perl::DIGEST_MD5 2.14 + Authen::SASL::Perl::EXTERNAL 2.14 + Authen::SASL::Perl::GSSAPI 0.05 + Authen::SASL::Perl::LOGIN 2.14 + Authen::SASL::Perl::Layer 2.14 + Authen::SASL::Perl::PLAIN 2.14 + requirements: + Digest::HMAC_MD5 0 + Digest::MD5 0 + ExtUtils::MakeMaker 6.42 + Test::More 0 + perl 5.005 B-Hooks-EndOfScope-0.21 pathname: E/ET/ETHER/B-Hooks-EndOfScope-0.21.tar.gz provides: @@ -310,6 +331,7 @@ DISTRIBUTIONS CGI::Simple::Standard 1.114 CGI::Simple::Util 1.114 requirements: + ExtUtils::MakeMaker 0 IO::Scalar 0 Test::More 0 CGI-Struct-1.21 @@ -1208,6 +1230,7 @@ DISTRIBUTIONS Config::Any::XML undef Config::Any::YAML undef requirements: + Config::General 2.47 Module::Pluggable::Object 3.6 Config-General-2.63 pathname: T/TL/TLINDEN/Config-General-2.63.tar.gz @@ -3688,6 +3711,7 @@ DISTRIBUTIONS IO::Prompt 0.997004 IO::Prompt::ReturnVal 0.997004 requirements: + ExtUtils::MakeMaker 0 IO::Handle 0 Term::ReadKey 0 Test::More 0 From e4baee66563c912927afce2bc093ee81a235b1d8 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 12 Apr 2019 11:58:40 -0400 Subject: [PATCH 118/892] check root of git checkout if config files cannot be found --- lib/MetaCPAN/Role/HasConfig.pm | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/lib/MetaCPAN/Role/HasConfig.pm b/lib/MetaCPAN/Role/HasConfig.pm index f0c506f1c..e28699724 100644 --- a/lib/MetaCPAN/Role/HasConfig.pm +++ b/lib/MetaCPAN/Role/HasConfig.pm @@ -5,8 +5,9 @@ use Moose::Role; use FindBin; use Config::ZOMG (); use MetaCPAN::Types qw(HashRef); +use Module::Runtime qw( require_module ); -# Done like this so can be required by a roles +# Done like this so can be required by a role sub config { return $_[0]->_config; } @@ -19,11 +20,29 @@ has _config => ( ); sub _build_config { + my $self = shift; + my $config = $self->_zomg("$FindBin::RealBin/.."); + return $config if $config; + + require_module('Git::Helpers'); + $config = $self->_zomg( Git::Helpers::checkout_root() ); + + return $config if $config; + + die "Couldn't find config file in $FindBin::RealBin/.. or " + . Git::Helpers::checkout_root(); +} + +sub _zomg { my $self = shift; - return Config::ZOMG->new( + my $path = shift; + + my $config = Config::ZOMG->new( name => 'metacpan_server', - path => "$FindBin::RealBin/..", - )->load; + path => $path, + ); + + return $config->open; } 1; From 6d6c31b60e22568d0ed89054caa7eb6aad7ceaa6 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 12 Apr 2019 11:59:05 -0400 Subject: [PATCH 119/892] Add skeleton SMTP config to metacpan_server.conf --- metacpan_server.conf | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/metacpan_server.conf b/metacpan_server.conf index 5558ea850..8edef4ec2 100644 --- a/metacpan_server.conf +++ b/metacpan_server.conf @@ -10,3 +10,10 @@ minion_dsn = postgresql:///minion_queue secret_key 8225b1874fdc431cedb1cf7d454a92b8fde3a5e6 + + + host smtp.fastmail.com + port 465 + username foo@metacpan.org + password seekrit + From b7afb90de3d2c907b57976bf06b51ad1a1d9bc56 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 12 Apr 2019 11:59:57 -0400 Subject: [PATCH 120/892] Use config for SMTP auth --- lib/MetaCPAN/Model/Email/PAUSE.pm | 20 ++++++++++++-------- metacpan_server_testing.conf | 7 +++++++ 2 files changed, 19 insertions(+), 8 deletions(-) diff --git a/lib/MetaCPAN/Model/Email/PAUSE.pm b/lib/MetaCPAN/Model/Email/PAUSE.pm index e4968e064..e5866a4d2 100644 --- a/lib/MetaCPAN/Model/Email/PAUSE.pm +++ b/lib/MetaCPAN/Model/Email/PAUSE.pm @@ -9,6 +9,8 @@ use Encode (); use MetaCPAN::Types qw( Object Uri ); use Try::Tiny qw( catch try ); +with('MetaCPAN::Role::HasConfig'); + has _author => ( is => 'ro', isa => Object, @@ -30,29 +32,31 @@ sub send { header => [ 'Content-Type' => 'text/plain; charset=utf-8', To => $self->_author->{email}->[0], - From => 'noreply@metacpan.org', + From => 'notifications@metacpan.org', Subject => 'Connect MetaCPAN with your PAUSE account', 'MIME-Version' => '1.0', ], body => $self->email_body, ); + my $config = $self->config->{smtp}; my $transport = Email::Sender::Transport::SMTP->new( { - host => 'smtp.fastmail.com', - port => 465, - sasl_username => 'foo', - sasl_password => 'bar', + debug => 1, + host => $config->{host}, + port => $config->{port}, + sasl_username => $config->{username}, + sasl_password => $config->{password}, + ssl => 1, } ); my $success = 0; try { - sendmail( $email, { transport => $transport } ); - $success = 1; + $success = sendmail( $email, { transport => $transport } ); } catch { - warn $_; + warn 'Could not send message: ' . $_; }; return $success; diff --git a/metacpan_server_testing.conf b/metacpan_server_testing.conf index 0dbc285e8..16008608b 100644 --- a/metacpan_server_testing.conf +++ b/metacpan_server_testing.conf @@ -22,3 +22,10 @@ github_key = foo github_secret = bar secret weak + + + host smtp.fastmail.com + port 465 + username foo@metacpan.org + password seekrit + From da915966c73a8f88bb8499a821bfd61285a19551 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 12 Apr 2019 23:40:13 -0400 Subject: [PATCH 121/892] Make email_body() private --- lib/MetaCPAN/Model/Email/PAUSE.pm | 4 ++-- lib/MetaCPAN/Server/Controller/Login/PAUSE.pm | 1 + t/model/email/pause.t | 4 +++- 3 files changed, 6 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Model/Email/PAUSE.pm b/lib/MetaCPAN/Model/Email/PAUSE.pm index e5866a4d2..90dab3fce 100644 --- a/lib/MetaCPAN/Model/Email/PAUSE.pm +++ b/lib/MetaCPAN/Model/Email/PAUSE.pm @@ -36,7 +36,7 @@ sub send { Subject => 'Connect MetaCPAN with your PAUSE account', 'MIME-Version' => '1.0', ], - body => $self->email_body, + body => $self->_email_body, ); my $config = $self->config->{smtp}; @@ -62,7 +62,7 @@ sub send { return $success; } -sub email_body { +sub _email_body { my $self = shift; my $name = $self->_author->name; my $uri = $self->_url; diff --git a/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm b/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm index 9ddb3966a..fb9fa515d 100644 --- a/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm +++ b/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm @@ -48,6 +48,7 @@ sub index : Path { my $url = $c->request->uri->clone; $url->query("code=$code"); + my $email = MetaCPAN::Model::Email::PAUSE->new( author => $author, url => $url, diff --git a/t/model/email/pause.t b/t/model/email/pause.t index 42e93c4c4..8b3e50438 100644 --- a/t/model/email/pause.t +++ b/t/model/email/pause.t @@ -41,7 +41,9 @@ my $email = MetaCPAN::Model::Email::PAUSE->new( url => URI->new('http://example.com/'), ); -ok( $email->send, 'send email' ); +ok( $email->_email_body, 'email_body' ); +ok( $email->send, 'send email' ); +diag $email->_email_body; my @messages = Email::Sender::Simple->default_transport->deliveries; is( @messages, 1, '1 message sent' ); From 287f1639e5340e5b43a513e6751fada23158d8f4 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 12 Apr 2019 23:42:41 -0400 Subject: [PATCH 122/892] Log an error if an email could not be sent --- lib/MetaCPAN/Server/Controller/Login/PAUSE.pm | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm b/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm index fb9fa515d..125fac102 100644 --- a/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm +++ b/lib/MetaCPAN/Server/Controller/Login/PAUSE.pm @@ -5,6 +5,7 @@ use warnings; use namespace::autoclean; use CHI (); +use Log::Contextual qw( :log :dlog ); use Moose; use Try::Tiny qw( catch try ); use MetaCPAN::Model::Email::PAUSE (); @@ -56,7 +57,10 @@ sub index : Path { my $sent = $email->send; - # XXX check return value of sending + if ( !$sent ) { + log_error { 'Could not send PAUSE email to ' . $author->pauseid }; + } + $c->controller('OAuth2')->redirect( $c, success => 'mail_sent' ); } } From 91e7532e7216a1bf4e99a196ac7c10e7cb3657ca Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Tue, 16 Apr 2019 00:11:26 +0200 Subject: [PATCH 123/892] sort cpanfile --- cpanfile | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/cpanfile b/cpanfile index 01c0d43d4..b69ba95dd 100644 --- a/cpanfile +++ b/cpanfile @@ -93,10 +93,10 @@ requires 'Log::Any::Adapter::Log4perl'; requires 'Log::Contextual'; requires 'Log::Log4perl'; requires 'Log::Log4perl::Appender::ScreenColoredLevels'; +requires 'MIME::Base64', '3.15'; requires 'MetaCPAN::Moose'; requires 'MetaCPAN::Pod::XHTML'; requires 'MetaCPAN::Role', '0.06'; -requires 'MIME::Base64', '3.15'; requires 'Minion', '>= 9.03'; requires 'Minion::Backend::SQLite'; requires 'Module::Load'; @@ -104,6 +104,8 @@ requires 'Module::Metadata', '1.000022'; requires 'Module::Pluggable'; requires 'Module::Runtime'; requires 'Mojo::Pg', '>= 4.08'; +requires 'Mojolicious::Plugin::MountPSGI', '0.14'; +requires 'Mojolicious::Plugin::OpenAPI'; requires 'Mojolicious::Plugin::Web::Auth', '0.000004'; requires 'Moose', ' >= 2.1403'; requires 'Moose::Role'; @@ -173,6 +175,7 @@ requires 'WWW::Mechanize::Cached', '1.50'; requires 'XML::Simple'; requires 'YAML', '1.15'; requires 'YAML::Syck', '1.29'; +requires 'YAML::XS', '0.67'; # Mojolicious::Plugin::OpenAPI YAML loading requires 'base'; requires 'feature'; requires 'namespace::autoclean'; @@ -181,9 +184,6 @@ requires 'strictures', 1; requires 'utf8'; requires 'version', '0.9901'; requires 'warnings'; -requires 'Mojolicious::Plugin::MountPSGI', '0.14'; -requires 'Mojolicious::Plugin::OpenAPI'; -requires 'YAML::XS', '0.67'; # Mojolicious::Plugin::OpenAPI YAML loading test_requires 'App::Prove'; test_requires 'CPAN::Faker', '0.010'; From 0a4940c38f066d5e65d34ef8ac69fb3ca7d4c834 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Tue, 16 Apr 2019 00:02:08 +0200 Subject: [PATCH 124/892] provide production log4perl config --- cpanfile | 4 ++++ cpanfile.snapshot | 17 +++++++++++++++++ log4perl_prod.conf | 18 ++++++++++++++++++ 3 files changed, 39 insertions(+) create mode 100644 log4perl_prod.conf diff --git a/cpanfile b/cpanfile index b69ba95dd..a75888522 100644 --- a/cpanfile +++ b/cpanfile @@ -93,6 +93,10 @@ requires 'Log::Any::Adapter::Log4perl'; requires 'Log::Contextual'; requires 'Log::Log4perl'; requires 'Log::Log4perl::Appender::ScreenColoredLevels'; +requires 'Log::Dispatch'; +requires 'Log::Dispatch::Syslog'; +requires 'Log::Log4perl::Catalyst'; +requires 'Log::Log4perl::Layout::JSON'; requires 'MIME::Base64', '3.15'; requires 'MetaCPAN::Moose'; requires 'MetaCPAN::Pod::XHTML'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index a5da179ed..d9c566fa0 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4225,6 +4225,23 @@ DISTRIBUTIONS File::Path 2.0606 File::Spec 0.82 Test::More 0.45 + Log-Log4perl-Layout-JSON-0.56 + pathname: M/MS/MSCHOUT/Log-Log4perl-Layout-JSON-0.56.tar.gz + provides: + Log::Log4perl::Layout::JSON 0.56 + requirements: + Carp 0 + Class::Tiny 0 + ExtUtils::MakeMaker 0 + JSON::MaybeXS 0 + Log::Log4perl 0 + Log::Log4perl::Layout 0 + Log::Log4perl::Layout::PatternLayout 0 + Log::Log4perl::Level 0 + parent 0 + perl 5.008 + strict 0 + warnings 0 Log-Message-0.08 pathname: B/BI/BINGOS/Log-Message-0.08.tar.gz provides: diff --git a/log4perl_prod.conf b/log4perl_prod.conf new file mode 100644 index 000000000..1adc5c024 --- /dev/null +++ b/log4perl_prod.conf @@ -0,0 +1,18 @@ +log4perl.rootLogger=WARN, OUTPUT, SYSLOG + +log4perl.appender.OUTPUT=Log::Log4perl::Appender::Screen +log4perl.appender.OUTPUT.stderr=1 + +log4perl.appender.OUTPUT.layout=PatternLayout +log4perl.appender.OUTPUT.layout.ConversionPattern=[%d] [%p] [%X{url}] %m%n + +log4perl.appender.SYSLOG=Log::Dispatch::Syslog +log4perl.appender.SYSLOG.ident = metacpan_api +log4perl.appender.SYSLOG.facility = local0 +log4perl.appender.SYSLOG.layout = Log::Log4perl::Layout::JSON +log4perl.appender.SYSLOG.layout.field.message = %m{chomp} +log4perl.appender.SYSLOG.layout.field.category = %c +log4perl.appender.SYSLOG.layout.field.class = %C +log4perl.appender.SYSLOG.layout.field.file = %F{1} +log4perl.appender.SYSLOG.layout.field.sub = %M{1} +log4perl.appender.SYSLOG.layout.include_mdc = 1 From c395227b012c79c802ca5ddd5c02019fc0ca369b Mon Sep 17 00:00:00 2001 From: Kang-min Liu Date: Thu, 18 Apr 2019 09:09:30 +0900 Subject: [PATCH 125/892] Set th "size" of resultset to be the same as input size. Otherwise Elasticsearch returns 10 hits by default. Since this 'by_author_and_names' routine is the backend of '/feed/recent' and responds up to 100 hits, this tweak can reduce the amount of roundtrip bteewn frontend and backend. --- lib/MetaCPAN/Query/Release.pm | 1 + 1 file changed, 1 insertion(+) diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index d31682868..34bbb71e6 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -363,6 +363,7 @@ sub by_author_and_names { # $releases: ArrayRef[ Dict[ author => Str, name => Str ] ] my $body = { + size => (0+ @$releases), query => { bool => { should => [ From a6cb9713492a740d9242e1fc8807eb8688439ae5 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 26 Apr 2019 11:21:11 +0100 Subject: [PATCH 126/892] configure Log::Contextual in psgi app --- app.psgi | 3 +++ lib/MetaCPAN/Model/Release.pm | 4 +--- 2 files changed, 4 insertions(+), 3 deletions(-) diff --git a/app.psgi b/app.psgi index 456c84e42..e7d03bdb8 100644 --- a/app.psgi +++ b/app.psgi @@ -6,6 +6,7 @@ use File::Basename (); use File::Path (); use File::Spec (); use Log::Log4perl (); +use Log::Contextual qw(set_logger); use Path::Tiny qw( path ); use Plack::App::Directory (); use Plack::App::URLMap (); @@ -36,6 +37,8 @@ BEGIN { $root_dir ); Log::Log4perl::init($log4perl_config); + set_logger(Log::Log4perl->get_logger('MetaCPAN::Server')); + package MetaCPAN::Server::WarnHandler; Log::Log4perl->wrapper_register(__PACKAGE__); my $logger = Log::Log4perl->get_logger; diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 67ccbfad1..9df8db32e 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -9,7 +9,7 @@ use CPAN::Meta (); use DateTime (); use File::Find (); use File::Spec (); -use Log::Contextual qw( :log :dlog ); +use Log::Contextual::Easy::Default qw( :log :dlog ); use MetaCPAN::Model::Archive; use MetaCPAN::Types qw(ArrayRef AbsFile Str); use MetaCPAN::Util qw( fix_version); @@ -19,8 +19,6 @@ use Path::Class (); use Parse::PMFile; use Try::Tiny qw( catch try ); -with 'MetaCPAN::Role::Logger'; - has archive => ( is => 'ro', isa => 'MetaCPAN::Model::Archive', From 8b199ebccc5066187bf38094a71cf3a522b1bd7b Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 26 Apr 2019 11:21:27 +0100 Subject: [PATCH 127/892] fix log4perl_file config option --- app.psgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.psgi b/app.psgi index e7d03bdb8..3a16f5e8a 100644 --- a/app.psgi +++ b/app.psgi @@ -18,7 +18,7 @@ my $config; BEGIN { $root_dir = File::Basename::dirname(__FILE__); $dev_mode = $ENV{PLACK_ENV} && $ENV{PLACK_ENV} eq 'development'; - $config = Config::ZOMG->new( + $config = Config::ZOMG->open( name => 'MetaCPAN::Server', path => $root_dir, ); From 544d11e378c0adeacb0b87189a5cfc1371382c89 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Fri, 26 Apr 2019 13:43:21 +0100 Subject: [PATCH 128/892] Revert "configure Log::Contextual in psgi app" This reverts commit a6cb9713492a740d9242e1fc8807eb8688439ae5. --- app.psgi | 3 --- lib/MetaCPAN/Model/Release.pm | 4 +++- 2 files changed, 3 insertions(+), 4 deletions(-) diff --git a/app.psgi b/app.psgi index 3a16f5e8a..da0ebdbb1 100644 --- a/app.psgi +++ b/app.psgi @@ -6,7 +6,6 @@ use File::Basename (); use File::Path (); use File::Spec (); use Log::Log4perl (); -use Log::Contextual qw(set_logger); use Path::Tiny qw( path ); use Plack::App::Directory (); use Plack::App::URLMap (); @@ -37,8 +36,6 @@ BEGIN { $root_dir ); Log::Log4perl::init($log4perl_config); - set_logger(Log::Log4perl->get_logger('MetaCPAN::Server')); - package MetaCPAN::Server::WarnHandler; Log::Log4perl->wrapper_register(__PACKAGE__); my $logger = Log::Log4perl->get_logger; diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 9df8db32e..67ccbfad1 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -9,7 +9,7 @@ use CPAN::Meta (); use DateTime (); use File::Find (); use File::Spec (); -use Log::Contextual::Easy::Default qw( :log :dlog ); +use Log::Contextual qw( :log :dlog ); use MetaCPAN::Model::Archive; use MetaCPAN::Types qw(ArrayRef AbsFile Str); use MetaCPAN::Util qw( fix_version); @@ -19,6 +19,8 @@ use Path::Class (); use Parse::PMFile; use Try::Tiny qw( catch try ); +with 'MetaCPAN::Role::Logger'; + has archive => ( is => 'ro', isa => 'MetaCPAN::Model::Archive', From c673e4bb458bad4f14b24bfd8d385d2794f1505d Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Thu, 4 Apr 2019 16:55:10 -0400 Subject: [PATCH 129/892] Take several (delayed) passes at setting the "latest" flag The previous behaviour was to run this job immediately after the indexing job finished, but this was proabably too early in almost all cases. Here we wait for a while and then try setting the flag. --- lib/MetaCPAN/Script/Release.pm | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index dc5d2fd60..2f00ac164 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -179,11 +179,25 @@ sub run { my $job_id = $self->_add_to_queue( index_release => [$file] => { priority => 3 } ); + # This is a hack to deal with the fact that we don't know exactly + # when 02packages gets updated. It should be about every 5 + # minutes. We could stop trying once something is already + # "latest", but some uploads will never be "latest". Trying this X + # times should be fairly cheap. If this doesn't work, there is a + # cleanup cron which can set the "latest" flag, if necessary. + if ( $self->latest ) { - $self->_add_to_queue( - index_latest => [ '--distribution', $d->dist ] => - { priority => 2, parents => [$job_id] } ); + for my $delay ( 150, 330, 600 ) { + $self->_add_to_queue( + index_latest => [ '--distribution', $d->dist ] => { + delay => $delay, + parents => [$job_id], + priority => 2, + } + ); + } } + } else { try { $self->import_archive($file) } From 93d6bba5cd21ea1b5a2b15be69c1508e69a32d97 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 26 Apr 2019 16:31:51 +0100 Subject: [PATCH 130/892] Don't default queue attempts to 1 --- lib/MetaCPAN/Script/Latest.pm | 7 +++++-- lib/MetaCPAN/Script/Queue.pm | 10 ++++++++-- lib/MetaCPAN/Script/Release.pm | 5 ++++- 3 files changed, 17 insertions(+), 5 deletions(-) diff --git a/lib/MetaCPAN/Script/Latest.pm b/lib/MetaCPAN/Script/Latest.pm index ca7bca55e..4fcec8ff8 100644 --- a/lib/MetaCPAN/Script/Latest.pm +++ b/lib/MetaCPAN/Script/Latest.pm @@ -47,8 +47,11 @@ sub _queue_latest { my $dist = shift || $self->distribution; log_info { "queueing " . $dist }; - $self->_add_to_queue( index_latest => - [ ( $self->force ? '--force' : () ), '--distribution', $dist ] ); + $self->_add_to_queue( + index_latest => + [ ( $self->force ? '--force' : () ), '--distribution', $dist ], + { attempts => 3 } + ); } sub run { diff --git a/lib/MetaCPAN/Script/Queue.pm b/lib/MetaCPAN/Script/Queue.pm index b2315f9bb..200a1d457 100644 --- a/lib/MetaCPAN/Script/Queue.pm +++ b/lib/MetaCPAN/Script/Queue.pm @@ -32,12 +32,18 @@ sub run { my $next = $rule->iter( $self->dir ); while ( defined( my $file = $next->() ) ) { - $self->_add_to_queue( index_release => [$file] ); + $self->_add_to_queue( + index_release => [$file], + { attempts => 3 } + ); } } if ( $self->_has_file ) { - $self->_add_to_queue( index_release => [ $self->file->stringify ] ); + $self->_add_to_queue( + index_release => [ $self->file->stringify ], + { attempts => 3 } + ); } } diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index 2f00ac164..87a5db1bd 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -177,7 +177,9 @@ sub run { if ( $self->queue ) { my $job_id = $self->_add_to_queue( - index_release => [$file] => { priority => 3 } ); + index_release => [$file], + { attempts => 3, priority => 3 } + ); # This is a hack to deal with the fact that we don't know exactly # when 02packages gets updated. It should be about every 5 @@ -190,6 +192,7 @@ sub run { for my $delay ( 150, 330, 600 ) { $self->_add_to_queue( index_latest => [ '--distribution', $d->dist ] => { + attempts => 3, delay => $delay, parents => [$job_id], priority => 2, From 11bc12a084c817dc86fb7a8b07b258a8dbb5611e Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Fri, 26 Apr 2019 16:59:09 +0100 Subject: [PATCH 131/892] Change Config::General to required This package is actually always needed not just for testing. Without it errors are generated. --- cpanfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cpanfile b/cpanfile index a75888522..29210d51b 100644 --- a/cpanfile +++ b/cpanfile @@ -30,6 +30,7 @@ requires 'CatalystX::Component::Traits'; requires 'CatalystX::Fastly::Role::Response', '0.06'; requires 'CatalystX::InjectComponent'; requires 'CatalystX::RoleApplicator'; +requires 'Config::General'; requires 'Config::ZOMG', '>=', '1.000000'; requires 'Const::Fast'; requires 'Cpanel::JSON::XS', '3.0115'; @@ -193,7 +194,6 @@ test_requires 'App::Prove'; test_requires 'CPAN::Faker', '0.010'; test_requires 'Code::TidyAll', '>= 0.47'; test_requires 'Code::TidyAll::Plugin::UniqueLines'; -test_requires 'Config::General'; test_requires 'Devel::Confess'; test_requires 'File::Copy'; test_requires 'HTTP::Cookies'; From 174be2732a24289b4b5fa99a604489c3c047f019 Mon Sep 17 00:00:00 2001 From: Shawn Sorichetti Date: Fri, 26 Apr 2019 17:00:16 +0100 Subject: [PATCH 132/892] Change to use metacpan-base Instead of using a base perl docker image, use the new metacpan-base image and build the api upon that. The result should be faster build times and smaller images. Environment variables have been introduced for being able to switch the server that is being executed in the image. Those variables are set in the `.env` file. --- .env | 2 ++ Dockerfile | 17 ++++++++--------- 2 files changed, 10 insertions(+), 9 deletions(-) create mode 100644 .env diff --git a/.env b/.env new file mode 100644 index 000000000..ff50b11fd --- /dev/null +++ b/.env @@ -0,0 +1,2 @@ +PGDB=db:5432 +API_SERVER=morbo -l http://*:5000 -w /metacpan-api --verbose diff --git a/Dockerfile b/Dockerfile index a7c421c62..34309c9e3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,18 +1,17 @@ -FROM perl:5.22 +FROM metacpan/metacpan-base:latest ENV PERL_MM_USE_DEFAULT=1 PERL_CARTON_PATH=/carton -COPY wait-for-it.sh / COPY cpanfile cpanfile.snapshot /metacpan-api/ WORKDIR /metacpan-api -RUN apt-get update \ - && apt-get install -y libgmp-dev rsync \ - && cpanm App::cpm \ - && cpm install -g Carton \ - && useradd -m metacpan-api -g users \ +# CPM installations of dependencies does not install or run tests. This is +# because the modules themselves have been tested, and the metacpan use of the +# modules is tested by the test suite. Removing the tests, reduces the overall +# size of the images. +RUN useradd -m metacpan-api -g users \ && mkdir /carton /CPAN \ - && cpm install -L /carton \ + && cpm install --without-test -L /carton \ && rm -fr /root/.cpanm /root/.perl-cpm /var/cache/apt/lists/* /tmp/* RUN chown -R metacpan-api:users /metacpan-api /carton /CPAN @@ -25,4 +24,4 @@ USER metacpan-api:users EXPOSE 5000 -CMD [ "/wait-for-it.sh", "db:5432", "--", "carton", "exec", "morbo", "-l", "http://*:5000/", "-w", "root", "./bin/api.pl"] +CMD /wait-for-it.sh ${PGDB} -- carton exec ${API_SERVER} ./bin/api.pl From ce5f6aa4a2340de85844125c1dab00ab75c68fa0 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 26 Apr 2019 18:17:31 +0100 Subject: [PATCH 133/892] Fix path that morbo is watching --- .env | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.env b/.env index ff50b11fd..cc4c34cad 100644 --- a/.env +++ b/.env @@ -1,2 +1,2 @@ PGDB=db:5432 -API_SERVER=morbo -l http://*:5000 -w /metacpan-api --verbose +API_SERVER=morbo -l http://*:5000 -w . --verbose From c35a09a186b0c651428a9f05837b1b9e0f81ecfa Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Fri, 26 Apr 2019 18:17:53 +0100 Subject: [PATCH 134/892] Remove carton from Dockerfile --- Dockerfile | 12 +++++------- 1 file changed, 5 insertions(+), 7 deletions(-) diff --git a/Dockerfile b/Dockerfile index 34309c9e3..dc19d691c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,6 +1,6 @@ FROM metacpan/metacpan-base:latest -ENV PERL_MM_USE_DEFAULT=1 PERL_CARTON_PATH=/carton +ENV PERL_MM_USE_DEFAULT=1 COPY cpanfile cpanfile.snapshot /metacpan-api/ WORKDIR /metacpan-api @@ -10,13 +10,11 @@ WORKDIR /metacpan-api # modules is tested by the test suite. Removing the tests, reduces the overall # size of the images. RUN useradd -m metacpan-api -g users \ - && mkdir /carton /CPAN \ - && cpm install --without-test -L /carton \ + && mkdir /CPAN \ + && cpm install --global --without-test \ && rm -fr /root/.cpanm /root/.perl-cpm /var/cache/apt/lists/* /tmp/* -RUN chown -R metacpan-api:users /metacpan-api /carton /CPAN - -VOLUME /carton +RUN chown -R metacpan-api:users /metacpan-api /CPAN VOLUME /CPAN @@ -24,4 +22,4 @@ USER metacpan-api:users EXPOSE 5000 -CMD /wait-for-it.sh ${PGDB} -- carton exec ${API_SERVER} ./bin/api.pl +CMD /wait-for-it.sh ${PGDB} -- ${API_SERVER} ./bin/api.pl From 6ef13c48904e2199356e98a29f0a89d1be570f29 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Fri, 26 Apr 2019 18:37:38 +0200 Subject: [PATCH 135/892] Script::Release: use S::Es count for checking existence --- lib/MetaCPAN/Script/Release.pm | 25 ++++++++++++++++--------- 1 file changed, 16 insertions(+), 9 deletions(-) diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index dc5d2fd60..787331eb6 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -161,15 +161,22 @@ sub run { my $d = CPAN::DistnameInfo->new($file); if ( $self->skip ) { - my $count = $self->index->type('release')->filter( - { - and => [ - { term => { archive => $d->filename } }, - { term => { author => $d->cpanid } }, - ] - } - )->raw->count; - if ($count) { + my $count = $self->es->count( + index => $self->index->name, + type => 'release', + body => { + query => { + bool => { + must => [ + { term => { archive => $d->filename } }, + { term => { author => $d->cpanid } }, + ] + } + } + }, + ); + + if ( $count->{count} ) { log_info {"Skipping $file"}; next; } From 7a2a12d36ac76f613678cf13ba99b7c4740237aa Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 10:46:21 +0100 Subject: [PATCH 136/892] Sort use statements in MetaCPAN::Server --- lib/MetaCPAN/Server.pm | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/lib/MetaCPAN/Server.pm b/lib/MetaCPAN/Server.pm index 670c94d7b..6e9e8fb73 100644 --- a/lib/MetaCPAN/Server.pm +++ b/lib/MetaCPAN/Server.pm @@ -4,15 +4,14 @@ use Moose; ## no critic (Modules::RequireEndWithOne) use Catalyst qw( +MetaCPAN::Role::Fastly::Catalyst ), '-Log=warn,error,fatal'; -use Log::Log4perl::Catalyst; - use CatalystX::RoleApplicator; +use Digest::SHA; use File::Temp qw( tempdir ); +use Log::Log4perl::Catalyst; +use Plack::Builder; use Plack::Middleware::ReverseProxy; use Plack::Middleware::ServerStatus::Lite; use Ref::Util qw( is_arrayref is_hashref ); -use Plack::Builder; -use Digest::SHA; extends 'Catalyst'; From 90aaf807426c1b5b7f0b1cd970f9beb5743d48e1 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 10:53:36 +0100 Subject: [PATCH 137/892] Remove ServerStatus::Lite plugin for now --- lib/MetaCPAN/Server.pm | 17 ----------------- 1 file changed, 17 deletions(-) diff --git a/lib/MetaCPAN/Server.pm b/lib/MetaCPAN/Server.pm index 6e9e8fb73..04c8b2d5d 100644 --- a/lib/MetaCPAN/Server.pm +++ b/lib/MetaCPAN/Server.pm @@ -122,23 +122,6 @@ sub app { enable 'Rewrite', rules => sub {s{^/?v\d+/}{}}; } - # Using an ES client against the API requires an index (/v0). - # In production nginx handles this. - - unless ( $ENV{HARNESS_ACTIVE} or $0 =~ /\.t$/ ) { - my $scoreboard = $class->path_to(qw(var tmp scoreboard)); - - # This may be a File object if it doesn't exist so change it, then make it. - my $dir = Path::Class::Dir->new( - ref $scoreboard ? $scoreboard->stringify : $scoreboard ); - $dir->mkpath unless -d $dir; - - enable 'ServerStatus::Lite', - path => '/server-status', - allow => ['127.0.0.1'], - scoreboard => $scoreboard, - ; - } $class->apply_default_middlewares( $class->psgi_app ); }; } From 555a2d0f4241a0d35b51a4697b5cdcc42bdda079 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 11:20:47 +0100 Subject: [PATCH 138/892] Run as root in Docker container --- Dockerfile | 7 +------ 1 file changed, 1 insertion(+), 6 deletions(-) diff --git a/Dockerfile b/Dockerfile index dc19d691c..523da8bc6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -9,17 +9,12 @@ WORKDIR /metacpan-api # because the modules themselves have been tested, and the metacpan use of the # modules is tested by the test suite. Removing the tests, reduces the overall # size of the images. -RUN useradd -m metacpan-api -g users \ - && mkdir /CPAN \ +RUN mkdir /CPAN \ && cpm install --global --without-test \ && rm -fr /root/.cpanm /root/.perl-cpm /var/cache/apt/lists/* /tmp/* -RUN chown -R metacpan-api:users /metacpan-api /CPAN - VOLUME /CPAN -USER metacpan-api:users - EXPOSE 5000 CMD /wait-for-it.sh ${PGDB} -- ${API_SERVER} ./bin/api.pl From 82e4358dc8f8077e9deab1235492609b12480024 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Fri, 26 Apr 2019 13:10:15 +0200 Subject: [PATCH 139/892] Removed unused Script::ReindexDist --- lib/MetaCPAN/Script/ReindexDist.pm | 112 ----------------------------- 1 file changed, 112 deletions(-) delete mode 100644 lib/MetaCPAN/Script/ReindexDist.pm diff --git a/lib/MetaCPAN/Script/ReindexDist.pm b/lib/MetaCPAN/Script/ReindexDist.pm deleted file mode 100644 index b3c315641..000000000 --- a/lib/MetaCPAN/Script/ReindexDist.pm +++ /dev/null @@ -1,112 +0,0 @@ -package MetaCPAN::Script::ReindexDist; - -# ABSTRACT: Reindex all releases of a distribution - -use strict; -use warnings; - -use Moose; -use MetaCPAN::Types qw( ArrayRef Bool Str ); - -with 'MetaCPAN::Role::Script', 'MooseX::Getopt'; - -has distribution => ( - is => 'ro', - isa => Str, - lazy => 1, - builder => '_build_distribution', -); - -sub _build_distribution { - my ($self) = @_; - - # First arg (after script name) is distribution name. - # Is there a better way to do this? - return $self->extra_argv->[1]; -} - -has releases => ( - is => 'ro', - isa => ArrayRef, - lazy => 1, - builder => '_build_releases', -); - -sub _build_releases { - my ($self) = @_; - return [ $self->index->type('release') - ->filter( { term => { distribution => $self->distribution } } ) - ->fields( [qw( download_url )] )->sort( ['date'] )->size(5000) - ->all ]; -} - -has sources => ( - is => 'ro', - isa => ArrayRef, - lazy => 1, - builder => '_build_sources', -); - -has prompt => ( - is => 'ro', - isa => Bool, - default => 1, - documentation => q{Prompt for confirmation (default true)}, -); - -sub _build_sources { - my ($self) = @_; - return [ map { $_->download_url } @{ $self->releases } ]; -} - -sub script { - my $self = shift; - local @ARGV = @_; - MetaCPAN::Script::Runner->run; -} - -sub run { - my ($self) = @_; - $self->confirm; - $self->script( - release => qw(--level debug --detect_backpan), - @{ $self->sources } - ); - $self->script( latest => '--distribution', $self->distribution ); -} - -sub confirm { - my ($self) = @_; - - die "No releases found for ${\ $self->distribution }\n" - if !@{ $self->releases }; - - print "Reindexing ${\ $self->distribution }\n", - ( map {" $_\n"} @{ $self->sources } ); - - if ( !$self->prompt ) { - return; - } - - print 'Continue? (y/n): '; - - my $confirmation = ; - - die "Aborted\n" - unless $confirmation =~ /^y/i; -} - -__PACKAGE__->meta->make_immutable; -1; - -__END__ - -=head1 SYNOPSIS - - # bin/metacpan reindexdist Foo-Bar - -=head1 DESCRIPTION - -Reindex all the releases of a named distribution. - -=cut From cf88b8e35e8922906663daf1679e300459695929 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 27 Apr 2019 12:59:50 +0200 Subject: [PATCH 140/892] Removed unused Script::Pagerank --- cpanfile | 1 - lib/MetaCPAN/Script/Pagerank.pm | 115 -------------------------------- 2 files changed, 116 deletions(-) delete mode 100644 lib/MetaCPAN/Script/Pagerank.pm diff --git a/cpanfile b/cpanfile index 29210d51b..a41836406 100644 --- a/cpanfile +++ b/cpanfile @@ -70,7 +70,6 @@ requires 'Find::Lib'; requires 'FindBin'; requires 'Gazelle'; requires 'Git::Helpers'; -requires 'Graph::Centrality::Pagerank'; requires 'Gravatar::URL'; requires 'HTML::Entities'; requires 'HTML::TokeParser::Simple'; diff --git a/lib/MetaCPAN/Script/Pagerank.pm b/lib/MetaCPAN/Script/Pagerank.pm deleted file mode 100644 index 9f9553b11..000000000 --- a/lib/MetaCPAN/Script/Pagerank.pm +++ /dev/null @@ -1,115 +0,0 @@ -package MetaCPAN::Script::Pagerank; - -use strict; -use warnings; - -use Graph::Centrality::Pagerank; -use Log::Contextual qw( :log ); -use Moose; - -with 'MetaCPAN::Role::Script', 'MooseX::Getopt'; - -sub run { - my $self = shift; - my $es = $self->es; - my $pr = Graph::Centrality::Pagerank->new(); - my @edges; - my $modules = $self->get_recent_modules; - - log_info {'Loading dependencies ...'}; - - my $scroll = $es->scroll_helper( - index => $self->index->name, - type => 'release', - body => { - query => { - filtered => { - query => { match_all => {} }, - filter => { - and => [ - { - term => { - 'dependency.phase' => 'runtime' - } - }, - { term => { status => 'latest' } }, - ] - } - } - }, - }, - scroll => '5m', - size => 1000, - ); - - log_info { $scroll->total, " recent releases found with dependencies" }; - - my $i = 0; - while ( my $release = $scroll->next ) { - foreach my $dep ( @{ $release->{_source}->{dependency} || [] } ) { - next if ( $dep->{phase} ne 'runtime' ); - my $dist = $modules->{ $dep->{module} }; - next unless ($dist); - $i++; - push( @edges, [ $release->{_source}->{name}, $dist ] ); - } - } - log_info { - "Calculating PageRankg with taking $i dependencies into account"; - }; - my $res = $pr->getPagerankOfNodes( listOfEdges => \@edges ); - my @sort = sort { $res->{$b} <=> $res->{$a} } keys %$res; - for ( 1 .. 500 ) { - my $mod = shift @sort; - print $mod, " ", $res->{$mod}, $/; - } -} - -sub get_recent_modules { - my $self = shift; - log_info {"Mapping modules to releases ..."}; - my $scroll = $self->es->scroll_helper( - index => $self->index->name, - type => 'file', - body => { - query => { - filtered => { - query => { match_all => {} }, - filter => { - and => [ - { term => { 'status' => 'latest' } }, - { term => { 'module.indexed' => 1 } }, - { term => { 'module.authorized' => 1 } }, - ] - } - } - } - }, - size => 1000, - fields => [ - qw(release distribution module.authorized module.indexed module.name) - ], - scroll => '1m', - ); - log_info { $scroll->total, " modules found" }; - my $result; - while ( my $file = $scroll->next ) { - next if ( $file->{fields}->{distribution} eq 'perl' ); - my $modules; - my $data; - for (qw(name authorized indexed)) { - $data->{$_} = $file->{fields}->{"module.$_"}; - $data->{$_} = [ $data->{$_} ] unless ( ref $data->{$_} ); - } - for ( my $i = 0; $i < @{ $data->{name} }; $i++ ) { - next - unless ( $data->{indexed}->[$i] eq "true" - && $data->{authorized}->[$i] eq "true" ); - $result->{ $data->{name}->[$i] } = $file->{fields}->{release}; - } - } - return $result; -} - -__PACKAGE__->meta->make_immutable; -1; From 73b36611c8dbf992dd9e423fcfe057adcf983117 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 27 Apr 2019 13:02:43 +0200 Subject: [PATCH 141/892] Removed unused Script::PerlMongers --- cpanfile | 2 - lib/MetaCPAN/Script/PerlMongers.pm | 105 ----------------------------- 2 files changed, 107 deletions(-) delete mode 100644 lib/MetaCPAN/Script/PerlMongers.pm diff --git a/cpanfile b/cpanfile index a41836406..419ca3478 100644 --- a/cpanfile +++ b/cpanfile @@ -174,8 +174,6 @@ requires 'Time::Local'; requires 'Try::Tiny', '0.24'; requires 'URI', '1.71'; requires 'URI::Escape'; -requires 'WWW::Mechanize', '1.75'; -requires 'WWW::Mechanize::Cached', '1.50'; requires 'XML::Simple'; requires 'YAML', '1.15'; requires 'YAML::Syck', '1.29'; diff --git a/lib/MetaCPAN/Script/PerlMongers.pm b/lib/MetaCPAN/Script/PerlMongers.pm deleted file mode 100644 index e83f99f1a..000000000 --- a/lib/MetaCPAN/Script/PerlMongers.pm +++ /dev/null @@ -1,105 +0,0 @@ -package MetaCPAN::Script::PerlMongers; - -use strict; -use warnings; -use feature 'say'; - -use Data::Dump qw( dump ); -use Find::Lib '../lib'; -use Moose; -use WWW::Mechanize::Cached; -use WWW::Mechanize; -use XML::Simple; - -with 'MetaCPAN::Role::Script'; - -sub index_perlmongers { - - my $self = shift; - my $groups = $self->get_pm_groups; - my @updates = (); - my @results = (); - - foreach my $group ( @{$groups} ) { - - my %update = ( - index => 'cpan', - type => 'perlmongers', - id => $group->{name}, - data => $group, - ); - - #push @updates, \%update; - my $result = $self->es->index(%update); - push @results, $result; - say dump($result); - } - - say dump( \@results ); - say dump( \@updates ); - - #my $result = $self->es->bulk( \@updates ); - return; - -} - -sub get_pm_groups { - - my $self = shift; - my $mech = WWW::Mechanize::Cached->new; - $mech->get('http://www.pm.org/groups/perl_mongers.xml'); - - my $xml = XMLin( $mech->content ); - my @groups = (); - my %groups = %{ $xml->{group} }; - - foreach my $pm_name ( sort keys %groups ) { - - my $group = $groups{$pm_name}; - my $date = delete $group->{date}; - - if ($date) { - my $date_key = $date->{type} . '_date'; - my $date_value = $date->{content}; - if ( $date_value =~ m{\A(\d\d\d\d)(\d\d)(\d\d)\z} ) { - $date_value = join "-", $1, $2, $3; - } - $group->{$date_key} = $date_value; - } - - my $id = delete $group->{id}; - $group->{pm_id} = $id; - - $pm_name =~ s{[\s\-]}{}gxms; - $group->{name} = $pm_name; - - push @groups, $group; - } - - return \@groups; - -} -__PACKAGE__->meta->make_immutable; -1; - -=pod - -=head1 SYNOPSIS - -Parse out PerlMonger Group info and add it to /cpan/perlmongers - - -=head2 get_pm_groups - -Fetches the authoritative XML file on PerlMongers groups, parses the XML and -returns an ARRAYREF of groups. - -=head2 index_perlmongers - -Adds/updates all PerlMongers groups to ElasticSearch. - -=head1 SOURCE - -L - -=cut From ac16fb118c988ffbbfa958b1457a3f061b266a67 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 12:35:26 +0100 Subject: [PATCH 142/892] Tidy --- lib/MetaCPAN/Query/Release.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index 34bbb71e6..6c0de5a69 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -363,7 +363,7 @@ sub by_author_and_names { # $releases: ArrayRef[ Dict[ author => Str, name => Str ] ] my $body = { - size => (0+ @$releases), + size => ( 0 + @$releases ), query => { bool => { should => [ From 398a715b2a33b7993aa8e204c79428bc3589c256 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 13:32:49 +0100 Subject: [PATCH 143/892] Tidy constants --- lib/MetaCPAN/Model/Search.pm | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index eed257425..6ade52c1b 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -25,9 +25,17 @@ has index => ( required => 1, ); -const my $RESULTS_PER_RUN => 200; -const my @ROGUE_DISTRIBUTIONS => - qw(kurila perl_debug perl_mlb perl-5.005_02+apache1.3.3+modperl pod2texi perlbench spodcxx Bundle-Everything); +const my $RESULTS_PER_RUN => 200; +const my @ROGUE_DISTRIBUTIONS => qw( + kurila + perl_debug + perl_mlb + perl-5.005_02+apache1.3.3+modperl + pod2texi + perlbench + spodcxx + Bundle-Everything +); sub _not_rogue { my @rogue_dists From df91b4216e7e278628838b4e6d0c23c6ad17667c Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 13:19:26 +0100 Subject: [PATCH 144/892] Don't index Acme::DependOnEverything --- lib/MetaCPAN/Document/File/Set.pm | 1 + lib/MetaCPAN/Model/Search.pm | 7 ++++--- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Document/File/Set.pm b/lib/MetaCPAN/Document/File/Set.pm index 72bfc7a69..446faa8fe 100644 --- a/lib/MetaCPAN/Document/File/Set.pm +++ b/lib/MetaCPAN/Document/File/Set.pm @@ -44,6 +44,7 @@ sub _build_query_favorite { } my @ROGUE_DISTRIBUTIONS = qw( + Acme-DependOnEverything Bundle-Everything kurila perl-5.005_02+apache1.3.3+modperl diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 6ade52c1b..46c46e5fd 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -27,14 +27,15 @@ has index => ( const my $RESULTS_PER_RUN => 200; const my @ROGUE_DISTRIBUTIONS => qw( + Acme-DependOnEverything + Bundle-Everything kurila + perl-5.005_02+apache1.3.3+modperl + perlbench perl_debug perl_mlb - perl-5.005_02+apache1.3.3+modperl pod2texi - perlbench spodcxx - Bundle-Everything ); sub _not_rogue { From 6fe8494c03445830e234c0691f59ae20e88b1ede Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 13:35:49 +0100 Subject: [PATCH 145/892] Tidy use statements --- lib/MetaCPAN/Model/Search.pm | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Model/Search.pm b/lib/MetaCPAN/Model/Search.pm index 46c46e5fd..007bbf109 100644 --- a/lib/MetaCPAN/Model/Search.pm +++ b/lib/MetaCPAN/Model/Search.pm @@ -3,14 +3,13 @@ package MetaCPAN::Model::Search; use MetaCPAN::Moose; use Const::Fast qw( const ); -use Log::Contextual qw( :log :dlog ); -use MooseX::StrictConstructor; use Cpanel::JSON::XS (); - use Hash::Merge qw( merge ); use List::Util qw( min uniq ); +use Log::Contextual qw( :log :dlog ); use MetaCPAN::Types qw( Object Str ); use MetaCPAN::Util qw( single_valued_arrayref_to_scalar ); +use MooseX::StrictConstructor; has es => ( is => 'ro', From cd61f97f41653dcafc5e7bdedefad6be3a6849fb Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 27 Apr 2019 15:07:56 +0200 Subject: [PATCH 146/892] Check if distribution exists before creating This is fixing the flood of errors exposed by the Log4Perl change. --- lib/MetaCPAN/Model/Release.pm | 12 +++++++++--- lib/MetaCPAN/Script/Release.pm | 1 + 2 files changed, 10 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 67ccbfad1..671884fc9 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -122,6 +122,7 @@ has status => ( isa => Str, ); +has es => ( is => 'ro' ); has bulk => ( is => 'ro' ); =head2 run @@ -221,11 +222,16 @@ sub _build_document { $document = $self->index->type('release')->put( $document, { refresh => 1 } ); - # create will die if the document already exists - eval { + # create distribution if doesn't exist + my $dist_count = $self->es->count( + index => 'cpan', + type => 'distribution', + body => { query => { term => { name => $self->distribution } } }, + ); + if ( !$dist_count->{count} ) { $self->index->type('distribution') ->put( { name => $self->distribution }, { create => 1 } ); - }; + } return $document; } diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index 9c21120c0..a0f78c566 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -228,6 +228,7 @@ sub _get_release_model { my $d = CPAN::DistnameInfo->new($archive_path); my $model = MetaCPAN::Model::Release->new( + es => $self->es, bulk => $bulk, distinfo => $d, file => $archive_path, From 7fb3422abad06d778d03d39d2799fb113b8b6789 Mon Sep 17 00:00:00 2001 From: Leo Lapworth Date: Sat, 27 Apr 2019 14:40:16 +0100 Subject: [PATCH 147/892] add deploy to docker image, just on master merge --- .travis.yml | 15 +++++++++++++++ deploy/build.sh | 14 ++++++++++++++ deploy/push.sh | 11 +++++++++++ deploy/vars.sh | 13 +++++++++++++ 4 files changed, 53 insertions(+) create mode 100755 deploy/build.sh create mode 100755 deploy/push.sh create mode 100755 deploy/vars.sh diff --git a/.travis.yml b/.travis.yml index c1df5fd4a..036d4ec00 100644 --- a/.travis.yml +++ b/.travis.yml @@ -21,6 +21,8 @@ env: - DEVEL_COVER_OPTIONS="-ignore,^local/" - PERL_CARTON_PATH=$HOME/local + + - DOCKER_IMAGE_NAME=metacpan-api matrix: - CPAN_RESOLVER=metadb PERL_CARTON_PATH=$HOME/no-snapshot HARNESS_VERBOSE=1 - CPAN_RESOLVER=snapshot @@ -83,6 +85,19 @@ after_success: services: - docker +## Build and push a docker image in production +deploy: + - provider: script + script: + - deploy/build.sh + on: + branch: master + - provider: script + script: + - deploy/push.sh + on: + branch: master + # caching /local should save about 5 minutes in module install time cache: directories: diff --git a/deploy/build.sh b/deploy/build.sh new file mode 100755 index 000000000..64013f312 --- /dev/null +++ b/deploy/build.sh @@ -0,0 +1,14 @@ +#!/bin/bash + +DEPLOY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +source "${DEPLOY_DIR}/vars.sh" + +# ## Go to where the docker file is +cd "${DEPLOY_DIR}/.." + +## Pull the latest docker file from docker hub if there is one +docker pull "$DOCKER_HUB_NAME" || true + +## Issue the build command, adding tags (from CONFIG.sh) +docker build --pull --cache-from "$DOCKER_HUB_NAME" --tag $DOCKER_HUB_NAME --tag $VERSION_TAG . diff --git a/deploy/push.sh b/deploy/push.sh new file mode 100755 index 000000000..bb24bd2ff --- /dev/null +++ b/deploy/push.sh @@ -0,0 +1,11 @@ +#!/bin/bash + +DEPLOY_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )" + +source "${DEPLOY_DIR}/vars.sh" + +cd "${DEPLOY_DIR}/.." + +docker login -u "$DOCKER_HUB_USER" -p "$DOCKER_HUB_PASSWD" + +docker push "$DOCKER_HUB_NAME" \ No newline at end of file diff --git a/deploy/vars.sh b/deploy/vars.sh new file mode 100755 index 000000000..7431405a6 --- /dev/null +++ b/deploy/vars.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +## Edit this +if [ -z $DOCKER_IMAGE_NAME ]; then + echo "DOCKER_IMAGE_NAME is not defined" + exit 1; +fi + +## Should not need to edit this +export DOCKER_HUB_NAME="metacpan/${DOCKER_IMAGE_NAME}" +export VERSION="${TRAVIS_BUILD_NUMBER:-UNKNOWN-BUILD-NUMBER}" +export VERSION_TAG="${DOCKER_HUB_NAME}:${VERSION}" + From b20ecdd2231c4637333cb14ff43a7a5776fc28b0 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 13:23:39 +0100 Subject: [PATCH 148/892] Upgrade Getopt::Long::Descriptive --- cpanfile | 1 + cpanfile.snapshot | 10 +++++----- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/cpanfile b/cpanfile index 419ca3478..0308a8c2f 100644 --- a/cpanfile +++ b/cpanfile @@ -69,6 +69,7 @@ requires 'File::stat'; requires 'Find::Lib'; requires 'FindBin'; requires 'Gazelle'; +requires 'Getopt::Long::Descriptive', '>= 0.103'; requires 'Git::Helpers'; requires 'Gravatar::URL'; requires 'HTML::Entities'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index d9c566fa0..095523bd3 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -3157,12 +3157,12 @@ DISTRIBUTIONS Stream::Buffered 0 Try::Tiny 0 perl 5.008001 - Getopt-Long-Descriptive-0.100 - pathname: R/RJ/RJBS/Getopt-Long-Descriptive-0.100.tar.gz + Getopt-Long-Descriptive-0.104 + pathname: R/RJ/RJBS/Getopt-Long-Descriptive-0.104.tar.gz provides: - Getopt::Long::Descriptive 0.100 - Getopt::Long::Descriptive::Opts 0.100 - Getopt::Long::Descriptive::Usage 0.100 + Getopt::Long::Descriptive 0.104 + Getopt::Long::Descriptive::Opts 0.104 + Getopt::Long::Descriptive::Usage 0.104 requirements: Carp 0 ExtUtils::MakeMaker 0 From 11f9721074906c9a3af59e3d0e829ac8c868dc6b Mon Sep 17 00:00:00 2001 From: Leo Lapworth Date: Sat, 27 Apr 2019 15:34:18 +0100 Subject: [PATCH 149/892] only deploy on the snapshot version --- .travis.yml | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/.travis.yml b/.travis.yml index 036d4ec00..a67c6c954 100644 --- a/.travis.yml +++ b/.travis.yml @@ -25,7 +25,7 @@ env: - DOCKER_IMAGE_NAME=metacpan-api matrix: - CPAN_RESOLVER=metadb PERL_CARTON_PATH=$HOME/no-snapshot HARNESS_VERBOSE=1 - - CPAN_RESOLVER=snapshot + - CPAN_RESOLVER=snapshot BUILD_DOCKER=yes matrix: allow_failures: @@ -92,11 +92,13 @@ deploy: - deploy/build.sh on: branch: master + condition: $BUILD_DOCKER -eq 'yes' - provider: script script: - deploy/push.sh on: branch: master + condition: $BUILD_DOCKER -eq 'yes' # caching /local should save about 5 minutes in module install time cache: From 83995ecf48f5d48bdd36ef1718f610f5dc3a73f9 Mon Sep 17 00:00:00 2001 From: Mickey Nasriachi Date: Sat, 27 Apr 2019 17:21:22 +0200 Subject: [PATCH 150/892] Fix size parameter reading for reverse_dependencieis (GH#736) --- lib/MetaCPAN/Query/Release.pm | 8 ++++---- lib/MetaCPAN/Server/Controller/ReverseDependencies.pm | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/lib/MetaCPAN/Query/Release.pm b/lib/MetaCPAN/Query/Release.pm index 6c0de5a69..a679b5b7e 100644 --- a/lib/MetaCPAN/Query/Release.pm +++ b/lib/MetaCPAN/Query/Release.pm @@ -666,7 +666,7 @@ sub requires { } sub reverse_dependencies { - my ( $self, $distribution, $page, $page_size, $sort ) = @_; + my ( $self, $distribution, $page, $page_size, $size, $sort ) = @_; # get the latest release of given distribution my $release = $self->_get_latest_release($distribution) || return; @@ -676,7 +676,7 @@ sub reverse_dependencies { # return releases depended on those modules return $self->_get_depended_releases( $modules, $page, $page_size, - $sort ); + $size, $sort ); } sub _get_latest_release { @@ -751,7 +751,7 @@ sub _fix_sort_value { } sub _get_depended_releases { - my ( $self, $modules, $page, $page_size, $sort ) = @_; + my ( $self, $modules, $page, $page_size, $size, $sort ) = @_; $page //= 1; $page_size //= 50; @@ -780,7 +780,7 @@ sub _get_depended_releases { ] } }, - size => $page_size, + size => $size || $page_size, from => $page * $page_size - $page_size, sort => $sort, } diff --git a/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm b/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm index fce90840c..1cc5db085 100644 --- a/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm +++ b/lib/MetaCPAN/Server/Controller/ReverseDependencies.pm @@ -15,7 +15,7 @@ sub dist : Path('dist') : Args(1) { my ( $self, $c, $dist ) = @_; $c->stash_or_detach( $c->model('CPAN::Release')->reverse_dependencies( - $dist, @{ $c->req->params }{qw< page page_size sort >} + $dist, @{ $c->req->params }{qw< page page_size size sort >} ) ); } From ca544109efe2b4b2dc8c47a193e3d09669ff7324 Mon Sep 17 00:00:00 2001 From: Leo Lapworth Date: Sat, 27 Apr 2019 17:37:16 +0100 Subject: [PATCH 151/892] make conditional deploy actually work --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index a67c6c954..8464da066 100644 --- a/.travis.yml +++ b/.travis.yml @@ -92,13 +92,13 @@ deploy: - deploy/build.sh on: branch: master - condition: $BUILD_DOCKER -eq 'yes' + condition: $BUILD_DOCKER = 'yes' - provider: script script: - deploy/push.sh on: branch: master - condition: $BUILD_DOCKER -eq 'yes' + condition: $BUILD_DOCKER = 'yes' # caching /local should save about 5 minutes in module install time cache: From dd5d9c847886306492afe22a4ebc97b29fb700ba Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 28 Apr 2019 10:57:05 +0100 Subject: [PATCH 152/892] Tweak index_latest jobs to coincide (a little bit) with PAUSE update schedule --- lib/MetaCPAN/Script/Release.pm | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/lib/MetaCPAN/Script/Release.pm b/lib/MetaCPAN/Script/Release.pm index a0f78c566..0821a2d62 100644 --- a/lib/MetaCPAN/Script/Release.pm +++ b/lib/MetaCPAN/Script/Release.pm @@ -188,15 +188,18 @@ sub run { { attempts => 3, priority => 3 } ); - # This is a hack to deal with the fact that we don't know exactly - # when 02packages gets updated. It should be about every 5 - # minutes. We could stop trying once something is already - # "latest", but some uploads will never be "latest". Trying this X - # times should be fairly cheap. If this doesn't work, there is a - # cleanup cron which can set the "latest" flag, if necessary. + # This is a hack to deal with the fact that we don't know exactly + # when 02packages gets updated. As of 2019-04-08, 02packages is + # updated via a cron which runs every 12 minutes, with the + # exception of one run which is skipped, resulting in a 24 minute + # gap. The run usually takes less than one minute. We could stop + # trying once something is already "latest", but some uploads will + # never be "latest". Trying this X times should be fairly cheap. + # If this doesn't work, there is a cleanup cron which can set the + # "latest" flag, if necessary. if ( $self->latest ) { - for my $delay ( 150, 330, 600 ) { + for my $delay ( 2 * 60, 7 * 60, 14 * 60, 26 * 60 ) { $self->_add_to_queue( index_latest => [ '--distribution', $d->dist ] => { attempts => 3, From 1a008d5bb72fee7e7415f9e1dc4cfc188bd2efba Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sat, 27 Apr 2019 16:04:30 +0100 Subject: [PATCH 153/892] Bump version of Mojolicious::Plugin::Web::Auth --- cpanfile | 2 +- cpanfile.snapshot | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/cpanfile b/cpanfile index 0308a8c2f..b014757d9 100644 --- a/cpanfile +++ b/cpanfile @@ -111,7 +111,7 @@ requires 'Module::Runtime'; requires 'Mojo::Pg', '>= 4.08'; requires 'Mojolicious::Plugin::MountPSGI', '0.14'; requires 'Mojolicious::Plugin::OpenAPI'; -requires 'Mojolicious::Plugin::Web::Auth', '0.000004'; +requires 'Mojolicious::Plugin::Web::Auth', '>= 0.17'; requires 'Moose', ' >= 2.1403'; requires 'Moose::Role'; requires 'Moose::Util'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 095523bd3..d42dea461 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -4916,10 +4916,10 @@ DISTRIBUTIONS requirements: ExtUtils::MakeMaker 0 JSON::Validator 2.14 - Mojolicious-Plugin-Web-Auth-0.15 - pathname: H/HA/HAYAJO/Mojolicious-Plugin-Web-Auth-0.15.tar.gz + Mojolicious-Plugin-Web-Auth-0.17 + pathname: H/HA/HAYAJO/Mojolicious-Plugin-Web-Auth-0.17.tar.gz provides: - Mojolicious::Plugin::Web::Auth 0.15 + Mojolicious::Plugin::Web::Auth 0.17 Mojolicious::Plugin::Web::Auth::Base undef Mojolicious::Plugin::Web::Auth::OAuth undef Mojolicious::Plugin::Web::Auth::OAuth2 undef @@ -4933,9 +4933,9 @@ DISTRIBUTIONS requirements: IO::Socket::SSL 1.77 Module::Build::Tiny 0.035 - Mojolicious 3.02 + Mojolicious 7.13 Net::OAuth 0.28 - perl 5.008005 + perl 5.010001 Moo-2.003003 pathname: H/HA/HAARG/Moo-2.003003.tar.gz provides: From f803d5654f15cb0cb7e2f6c97df1c8d5a9b3c9c0 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 28 Apr 2019 11:20:09 +0100 Subject: [PATCH 154/892] Add dummy oauth config to metacpan_server.conf --- metacpan_server.conf | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) diff --git a/metacpan_server.conf b/metacpan_server.conf index 8edef4ec2..714c402c2 100644 --- a/metacpan_server.conf +++ b/metacpan_server.conf @@ -17,3 +17,20 @@ minion_dsn = postgresql:///minion_queue username foo@metacpan.org password seekrit + + + + key = seekrit + secret = seekrit + + + key = seekrit + secret = seekrit + + + key = seekrit + secret = seekrit + + + +front_end_url = http://0.0.0.0:5001 From 65810259fd7491358152d9c2e316a7d6ebc30e24 Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 28 Apr 2019 11:21:04 +0100 Subject: [PATCH 155/892] Add oauth routes to MetaCPAN::API --- lib/MetaCPAN/API.pm | 101 +++++++++++++++++++++++++++++++++----------- 1 file changed, 76 insertions(+), 25 deletions(-) diff --git a/lib/MetaCPAN/API.pm b/lib/MetaCPAN/API.pm index 3b073e4de..c0589e269 100644 --- a/lib/MetaCPAN/API.pm +++ b/lib/MetaCPAN/API.pm @@ -104,35 +104,28 @@ sub _set_up_routes { my $r = $self->routes; - $self->plugin( - 'Web::Auth', - module => 'Github', - key => $self->config->{github_key}, - secret => $self->config->{github_secret}, - on_finished => sub { - my ( $c, $access_token, $account_info ) = @_; - my $login = $account_info->{login}; - if ( $self->_is_admin($login) ) { - $c->session( username => $login ); - $c->redirect_to('/admin'); - return; - } - return $c->render( - text => "$login is not authorized to access this application", - status => 403 - ); - }, - ); - my $admin = $r->under( '/admin' => sub { - my $c = shift; - return 1 if $self->_is_admin( $c->session('username') ); + my $c = shift; + my $username = $c->session('github_username'); + if ( $self->_is_admin($username) ) { + return 1; + } + + # Direct non-admins away from the app + elsif ($username) { + $c->redirect_to('https://metacpan.org/'); + return 0; + } + + # This is possibly a logged out admin $c->redirect_to('/auth/github/authenticate'); return 0; } ); + $self->_set_up_oauth_routes; + $admin->get('home')->to('admin#home')->name('admin-home'); $admin->post('enqueue')->to('queue#enqueue')->name('enqueue'); $admin->post('search-identities')->to('admin#search_identities') @@ -165,9 +158,8 @@ sub _set_up_routes { } sub _is_admin { - my $self = shift; - my $username - = shift || ( $ENV{HARNESS_ACTIVE} ? $ENV{FORCE_ADMIN_AUTH} : () ); + my $self = shift; + my $username = $ENV{HARNESS_ACTIVE} ? $ENV{FORCE_ADMIN_AUTH} : shift; return 0 unless $username; my @admins = ( @@ -202,4 +194,63 @@ sub _build_db_params { die "Unsupported Database in dsn: " . $self->config->{minion_dsn}; } +sub _set_up_oauth_routes { + my $self = shift; + + my $oauth = $self->config->{oauth}; + + # We could do better DRY here, but it might be more complicated than it's + # worth + + $self->plugin( + 'Web::Auth', + module => 'Github', + key => $oauth->{github}->{key}, + secret => $oauth->{github}->{secret}, + user_info => 1, + on_finished => sub { + my ( $c, $access_token, $account_info ) = @_; + my $username = $account_info->{login}; + $c->session( is_logged_in => 1 ); + $c->session( github_username => $username ); + if ( $self->_is_admin($username) ) { + $c->session( gitnub_username => $username ); + $c->redirect_to('/admin'); + return; + } + $c->redirect_to( $self->config->{front_end_url} ); + }, + ); + + $self->plugin( + 'Web::Auth', + module => 'Google', + key => $oauth->{google}->{key}, + secret => $oauth->{google}->{secret}, + user_info => 1, + on_finished => sub { + my ( $c, $access_token, $account_info ) = @_; + my $username = $account_info->{login}; + $c->session( is_logged_in => 1 ); + $c->session( google_username => $username ); + $c->redirect_to( $self->config->{front_end_url} ); + }, + ); + + $self->plugin( + 'Web::Auth', + module => 'Twitter', + key => $oauth->{twitter}->{key}, + secret => $oauth->{twitter}->{secret}, + user_info => 1, + on_finished => sub { + my ( $c, $access_token, $access_secret, $account_info ) = @_; + my $username = $account_info->{screen_name}; + $c->session( is_logged_in => 1 ); + $c->session( twitter_username => $username ); + $c->redirect_to( $self->config->{front_end_url} ); + }, + ); +} + 1; From b682357a79b51d2acb8c8a115055d16cfa1b078c Mon Sep 17 00:00:00 2001 From: Olaf Alders Date: Sun, 28 Apr 2019 13:14:49 +0100 Subject: [PATCH 156/892] /bin/prove no longer needs to use Carton --- bin/prove | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/bin/prove b/bin/prove index 3f4cf5e63..81356cab7 100755 --- a/bin/prove +++ b/bin/prove @@ -5,4 +5,4 @@ export ES=${ES_TEST:-"localhost:9900"} export METACPAN_SERVER_CONFIG_LOCAL_SUFFIX=testing unset ES_SCRIPT_INDEX -`dirname "$0"`/run prove -It/lib -lvr "$@" +prove -It/lib -lvr "$@" From 5269e6ace513d5f52ff4a40df27ab24fcc678eb7 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 28 Apr 2019 15:34:54 +0100 Subject: [PATCH 157/892] remove Log::Any setup to quiet logs It will be reintroduced later with configuration cleanups. --- lib/MetaCPAN/Model.pm | 3 --- 1 file changed, 3 deletions(-) diff --git a/lib/MetaCPAN/Model.pm b/lib/MetaCPAN/Model.pm index fc96fe440..782045640 100644 --- a/lib/MetaCPAN/Model.pm +++ b/lib/MetaCPAN/Model.pm @@ -4,9 +4,6 @@ package MetaCPAN::Model; use Moose; use ElasticSearchX::Model; -use Log::Any::Adapter; - -Log::Any::Adapter->set('Log4perl'); analyzer lowercase => ( tokenizer => 'keyword', From 3e0961566e917213553b73992280afb27be603e8 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 28 Apr 2019 13:22:55 +0100 Subject: [PATCH 158/892] remove Facebook login --- cpanfile | 1 - cpanfile.snapshot | 79 ------------------- lib/MetaCPAN/Server/Controller/Login.pm | 1 - .../Server/Controller/Login/Facebook.pm | 39 --------- templates/admin/identity_search_form.html.ep | 1 - 5 files changed, 121 deletions(-) delete mode 100644 lib/MetaCPAN/Server/Controller/Login/Facebook.pm diff --git a/cpanfile b/cpanfile index 0308a8c2f..2a43486de 100644 --- a/cpanfile +++ b/cpanfile @@ -57,7 +57,6 @@ requires 'Encoding::FixLatin'; requires 'Encoding::FixLatin::XS'; requires 'Exporter'; requires 'ExtUtils::HasCompiler'; -requires 'Facebook::Graph'; requires 'File::Basename'; requires 'File::Find'; requires 'File::Find::Rule'; diff --git a/cpanfile.snapshot b/cpanfile.snapshot index 095523bd3..06caf917e 100644 --- a/cpanfile.snapshot +++ b/cpanfile.snapshot @@ -2887,42 +2887,6 @@ DISTRIBUTIONS Module::CPANfile 0 Test::More 0.88 version 0.76 - Facebook-Graph-1.1204 - pathname: R/RI/RIZEN/Facebook-Graph-1.1204.tar.gz - provides: - Facebook::Graph 1.1204 - Facebook::Graph::AccessToken 1.1204 - Facebook::Graph::AccessToken::Response 1.1204 - Facebook::Graph::Authorize 1.1204 - Facebook::Graph::BatchRequests 1.1204 - Facebook::Graph::Page::Feed 1.1204 - Facebook::Graph::Picture 1.1204 - Facebook::Graph::Publish 1.1204 - Facebook::Graph::Publish::Checkin 1.1204 - Facebook::Graph::Publish::Comment 1.1204 - Facebook::Graph::Publish::PageTab 1.1204 - Facebook::Graph::Publish::Photo 1.1204 - Facebook::Graph::Publish::Post 1.1204 - Facebook::Graph::Publish::RSVPAttending 1.1204 - Facebook::Graph::Publish::RSVPDeclined 1.1204 - Facebook::Graph::Publish::RSVPMaybe 1.1204 - Facebook::Graph::Query 1.1204 - Facebook::Graph::Request 1.1204 - Facebook::Graph::Response 1.1204 - Facebook::Graph::Role::Uri 1.1204 - Facebook::Graph::Session 1.1204 - requirements: - DateTime 0.61 - DateTime::Format::Strptime 1.4000 - ExtUtils::MakeMaker 0 - JSON 2.16 - LWP::Protocol::https 6.06 - LWP::UserAgent 6.13 - MIME::Base64::URLSafe 0.01 - Moo 0 - Ouch 0.0400 - Test::More 0 - URI 1.54 Fennec-Lite-0.004 pathname: E/EX/EXODIST/Fennec-Lite-0.004.tar.gz provides: @@ -4313,13 +4277,6 @@ DISTRIBUTIONS perl 5.008 strict 0 warnings 0 - MIME-Base64-URLSafe-0.01 - pathname: K/KA/KAZUHO/MIME-Base64-URLSafe-0.01.tar.gz - provides: - MIME::Base64::URLSafe 0.01 - requirements: - ExtUtils::MakeMaker 0 - MIME::Base64 0 MIME-Charset-1.012.2 pathname: N/NE/NEZUMI/MIME-Charset-1.012.2.tar.gz provides: @@ -6301,17 +6258,6 @@ DISTRIBUTIONS parent 0 perl 5.008005 version 0.9912 - Ouch-0.0500 - pathname: R/RI/RIZEN/Ouch-0.0500.tar.gz - provides: - Ouch 0.0500 - requirements: - Carp 0 - ExtUtils::MakeMaker 0 - Test::More 0 - Test::Trap 0 - overload 0 - parent 0 PAUSE-Permissions-0.17 pathname: N/NE/NEILB/PAUSE-Permissions-0.17.tar.gz provides: @@ -8489,31 +8435,6 @@ DISTRIBUTIONS Test::SharedFork 0.29 Time::HiRes 0 perl 5.008001 - Test-Trap-v0.3.3 - pathname: E/EB/EBHANSSEN/Test-Trap-v0.3.3.tar.gz - provides: - Test::Trap v0.3.3 - Test::Trap::Builder v0.3.3 - Test::Trap::Builder::PerlIO v0.3.3 - Test::Trap::Builder::SystemSafe v0.3.3 - Test::Trap::Builder::TempFile v0.3.3 - requirements: - Carp 0 - Data::Dump 0 - Exporter 0 - File::Temp 0 - IO::Handle 0 - Module::Build 0 - Test::Builder 0 - Test::More 0 - Test::Tester 0.107 - base 0 - constant 0 - lib 0 - perl v5.6.2 - strict 0 - version 0 - warnings 0 Test-Vars-0.014 pathname: D/DR/DROLSKY/Test-Vars-0.014.tar.gz provides: diff --git a/lib/MetaCPAN/Server/Controller/Login.pm b/lib/MetaCPAN/Server/Controller/Login.pm index aa94770e7..d4b7af597 100644 --- a/lib/MetaCPAN/Server/Controller/Login.pm +++ b/lib/MetaCPAN/Server/Controller/Login.pm @@ -3,7 +3,6 @@ package MetaCPAN::Server::Controller::Login; use strict; use warnings; -use Facebook::Graph; use Cpanel::JSON::XS; use Moose; diff --git a/lib/MetaCPAN/Server/Controller/Login/Facebook.pm b/lib/MetaCPAN/Server/Controller/Login/Facebook.pm deleted file mode 100644 index a25495d0f..000000000 --- a/lib/MetaCPAN/Server/Controller/Login/Facebook.pm +++ /dev/null @@ -1,39 +0,0 @@ -package MetaCPAN::Server::Controller::Login::Facebook; - -use strict; -use warnings; - -use Facebook::Graph; -use Moose; - -BEGIN { extends 'MetaCPAN::Server::Controller::Login' } - -has [qw(consumer_key consumer_secret)] => ( - is => 'ro', - required => 1, -); - -sub index : Path { - my ( $self, $c ) = @_; - - my $callback = $c->request->uri->clone; - $callback->query(undef); - my $fb = Facebook::Graph->new( - app_id => $self->consumer_key, - secret => $self->consumer_secret, - postback => $callback, - ); - - if ( my $code = $c->req->params->{code} ) { - my $token = eval { $fb->request_access_token($code)->token } - or $c->controller('OAuth2')->redirect( $c, error => 'token' ); - my $data = $fb->query->find('me')->request->as_hashref; - $self->update_user( $c, facebook => $data->{id}, $data ); - } - else { - my $auth_url = $fb->authorize->uri_as_string; - $c->res->redirect($auth_url); - } -} - -1; diff --git a/templates/admin/identity_search_form.html.ep b/templates/admin/identity_search_form.html.ep index 06bcfb737..43fbf5e77 100644 --- a/templates/admin/identity_search_form.html.ep +++ b/templates/admin/identity_search_form.html.ep @@ -1,7 +1,6 @@
- - - - - - - Identity value: - - -
diff --git a/templates/admin/index.html.ep b/templates/admin/index.html.ep deleted file mode 100644 index a39edf7f1..000000000 --- a/templates/admin/index.html.ep +++ /dev/null @@ -1,8 +0,0 @@ - diff --git a/templates/admin/search_identities.html.ep b/templates/admin/search_identities.html.ep deleted file mode 100644 index 3428abc44..000000000 --- a/templates/admin/search_identities.html.ep +++ /dev/null @@ -1,3 +0,0 @@ -display results below: -<%= stash('user_data')->{identity}[0]{name} %> -<%= stash('user_data')->{identity}[0]{key} %> diff --git a/templates/layouts/default.html.ep b/templates/layouts/default.html.ep deleted file mode 100644 index fbf9c181c..000000000 --- a/templates/layouts/default.html.ep +++ /dev/null @@ -1,5 +0,0 @@ -

-
MetaCPAN Admin
-

- -<%= content %> diff --git a/templates/queue/index_release.html.ep b/templates/queue/index_release.html.ep deleted file mode 100644 index e69de29bb..000000000 From 6777b50e7cfb33a78775d54c2e7b145c7fda57e5 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:22:36 +0200 Subject: [PATCH 871/892] stop testing Minion setup Our tests for minion don't really check much, and add a dependency on Postgres. Eventually these tests can return in ingest once it is ready. --- t/api/queue.t | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/t/api/queue.t b/t/api/queue.t index b70e051e9..492244b38 100644 --- a/t/api/queue.t +++ b/t/api/queue.t @@ -2,10 +2,10 @@ use strict; use warnings; use lib 't/lib'; +use Test::More skip_all => 'disabling Minion tests to avoid needing postgres'; use MetaCPAN::DarkPAN (); use Path::Tiny qw( path ); use Test::Mojo; -use Test::More; my $t = Test::Mojo->new('MetaCPAN::API'); my $app = $t->app; From fe3e94f711201cd6388276ce7a4ed09740f6b7f0 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:24:36 +0200 Subject: [PATCH 872/892] add ${VAR} replacement support to config This supports ${VAR}, ${VAR:-default}, and ${VAR:+set} variable replacements to allow overridable config variables. --- lib/MetaCPAN/Server/Config.pm | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Config.pm b/lib/MetaCPAN/Server/Config.pm index 4bf51f97a..a8b032c7d 100644 --- a/lib/MetaCPAN/Server/Config.pm +++ b/lib/MetaCPAN/Server/Config.pm @@ -36,7 +36,28 @@ sub _zomg { my $v = Data::Visitor::Callback->new( plain_value => sub { return unless defined $_; - s{__HOME__}{$root}ge; + s{ + (__HOME__) + | + (\$\{([^\}]+)\}) + }{ + defined $1 ? $root + : defined $2 ? do { + my $var = $3; + if ($var =~ s{:-(.*)}{}) { + my $sub = $1; + $ENV{$var} // $1; + } + elsif ($var =~ s{:\+(.*)}{}) { + my $sub = $1; + $ENV{$var} ? $sub : ''; + } + else { + $ENV{$var} // ''; + } + } + : '' + }gex; } ); $v->visit($c); From 902192df1bcbce3a5640f6990ef148c569e01385 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:27:00 +0200 Subject: [PATCH 873/892] add testing profile to docker compose --- docker-compose.yml | 47 ++++++++++++++++++++++++++++++++++++ metacpan_server_testing.yaml | 2 +- 2 files changed, 48 insertions(+), 1 deletion(-) diff --git a/docker-compose.yml b/docker-compose.yml index 965fbaa92..99066703d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -15,3 +15,50 @@ services: watch: - path: ./cpanfile action: rebuild + + api-test: + profiles: + - test + depends_on: + elasticsearch-test: + condition: service_healthy + build: + context: . + target: test + environment: + NET_ASYNC_HTTP_MAXCONNS: 1 + COLUMNS: 80 + ES: http://elasticsearch-test:9200 + HARNESS_ACTIVE: 1 + # Instantiate Catalyst models using metacpan_server_testing.conf + METACPAN_SERVER_CONFIG_LOCAL_SUFFIX: testing + MINICPAN: /CPAN + DEVEL_COVER_OPTIONS: +ignore,^t/|^test-data/|^etc/|^local/ + networks: + - elasticsearch + volumes: + - type: volume + source: elasticsearch-test + target: /usr/share/elasticsearch/data + + elasticsearch-test: + profiles: + - test + platform: linux/amd64 + image: elasticsearch:2.4 + environment: + - discovery.type=single-node + healthcheck: + timeout: 5s + start_period: 60s + test: ["CMD", "curl", "--fail", "http://localhost:9200/_cluster/health?wait_for_status=yellow&timeout=5s"] + ports: + - "9200" + networks: + - elasticsearch + +networks: + elasticsearch: + +volumes: + elasticsearch-test: diff --git a/metacpan_server_testing.yaml b/metacpan_server_testing.yaml index f47e101d5..23b6e4406 100644 --- a/metacpan_server_testing.yaml +++ b/metacpan_server_testing.yaml @@ -8,7 +8,7 @@ source_base: var/t/tmp/source elasticsearch_servers: client: '2_0::Direct' - nodes: http://elasticsearch_test:9200 + nodes: ${ES:-http://elasticsearch_test:9200} minion_dsn: "postgresql://metacpan:t00lchain@pghost:5432/minion_queue" From 62652423cb5507eb80189520ed8d908f4b68fa61 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 09:02:02 +0200 Subject: [PATCH 874/892] update docker build and push to use our standard setup Use a single workflow for docker building and pushing, and use the reusable action to make it standardized. Leaving in a commented out step to update in k8s. Once it is set up, we can uncomment it. --- .github/workflows/build-container.yml | 46 +++++++++++++++++++ .../workflows/build-deployment-container.yml | 26 ----------- .../workflows/build-production-container.yml | 24 ---------- 3 files changed, 46 insertions(+), 50 deletions(-) create mode 100644 .github/workflows/build-container.yml delete mode 100644 .github/workflows/build-deployment-container.yml delete mode 100644 .github/workflows/build-production-container.yml diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml new file mode 100644 index 000000000..817561902 --- /dev/null +++ b/.github/workflows/build-container.yml @@ -0,0 +1,46 @@ +name: Build container +on: + push: + branches: + - master + - staging + - prod + pull_request: + types: [opened, synchronize, labeled] + branches: + - master + workflow_dispatch: +jobs: + docker-build: + if: github.event_name != 'pull_request' || contains(github.event.pull_request.labels.*.name, 'build-container') + runs-on: ubuntu-22.04 + name: Docker Build and Push + steps: + - name: Generate Auth Token + uses: actions/create-github-app-token@v2 + id: app-token + with: + app-id: ${{ secrets.APP_ID }} + private-key: ${{ secrets.APP_PRIVATE_KEY }} + owner: metacpan + - uses: actions/checkout@v4 + with: + token: ${{ steps.app-token.outputs.token }} + - uses: metacpan/metacpan-actions/docker-build-push@master + id: build-push + with: + docker_hub_username: ${{ secrets.DOCKER_HUB_USER }} + docker_hub_password: ${{ secrets.DOCKER_HUB_TOKEN }} + ghcr_username: ${{ github.repository_owner }} + ghcr_password: ${{ secrets.GITHUB_TOKEN }} + test-target: test +# - name: Update deployed image +# if: steps.find-tag-names.outputs.latest +# uses: metacpan/metacpan-actions/update-deployed-tag:master +# with: +# token: ${{ steps.app-token.outputs.token }} +# app: api +# environment: prod +# base-tag: ${{ steps.find-tag-names.outputs.latest }} +# tag: ${{ steps.find-tag-names.outputs.sha }} + diff --git a/.github/workflows/build-deployment-container.yml b/.github/workflows/build-deployment-container.yml deleted file mode 100644 index 721d46dc7..000000000 --- a/.github/workflows/build-deployment-container.yml +++ /dev/null @@ -1,26 +0,0 @@ ---- -name: Build deployment container -on: - push: - branches: - - prod - - staging - workflow_dispatch: -jobs: - docker: - runs-on: ubuntu-22.04 - name: Docker push SHA - steps: - - uses: actions/checkout@v4 - - name: docker build - run: docker build . -t metacpan/metacpan-api:$GITHUB_SHA - - name: run Perl tests - run: docker run -i metacpan/metacpan-api carton exec prove -lr --jobs 2 t - - name: Log in to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: push build to Docker hub - run: docker push metacpan/metacpan-api:$GITHUB_SHA - diff --git a/.github/workflows/build-production-container.yml b/.github/workflows/build-production-container.yml deleted file mode 100644 index 584f9b54f..000000000 --- a/.github/workflows/build-production-container.yml +++ /dev/null @@ -1,24 +0,0 @@ ---- -name: Build Production Container -on: - push: - branches: - - master - workflow_dispatch: - -jobs: - docker: - runs-on: ubuntu-22.04 - name: Docker push latest - steps: - - uses: actions/checkout@v4 - - name: docker build - run: docker build . -t metacpan/metacpan-api:latest - - name: Login to Docker Hub - uses: docker/login-action@v3 - with: - username: ${{ secrets.DOCKER_HUB_USER }} - password: ${{ secrets.DOCKER_HUB_TOKEN }} - - name: push build to Docker hub - run: docker push metacpan/metacpan-api:latest - if: success() && github.ref == 'refs/heads/master' From 74b9d098920917f2d8e9d1f62ea37b66157412d3 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sat, 28 Jun 2025 11:27:12 +0200 Subject: [PATCH 875/892] update circleci config for new docker compose setup This repo now has its own docker compose config, so it doesn't need to pull in a separate repo for testing. This simplifies the CI setup, and makes future updates easier as they don't have to coordinate across multiple repos. --- .circleci/config.yml | 62 ++++++++------------------------------------ 1 file changed, 11 insertions(+), 51 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index dc0c7ae68..0aa915976 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -15,64 +15,24 @@ jobs: resource_class: large steps: - run: - name: docker-compose version - command: docker-compose --version + name: docker compose version + command: docker compose version + - checkout - run: + name: create coverage directory command: | - git clone https://github.com/metacpan/metacpan-docker.git - cd metacpan-docker - name: metacpan-docker checkout - - checkout: - path: metacpan-docker/src/metacpan-api + mkdir cover_db + chmod o+w cover_db - run: + name: docker compose build command: | - pushd metacpan-docker - ./bin/metacpan-docker init - name: clone missing repositories + docker compose --profile test build api-test - run: - command: | - pushd metacpan-docker - docker-compose build --build-arg CPM_ARGS='--with-test' api_test - name: compose build - - run: - command: | - pushd metacpan-docker - ./bin/metacpan-docker init - docker-compose --verbose up -d api_test - name: compose up - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test cpm install -g Devel::Cover - name: install Devel::Cover - # Since we're running docker-compose -d, we don't actually know if - # Elasticsearch is available at the time this build step begins. We - # probably need to wait for it here, so we'll add our own check. - - run: - command: | - pushd metacpan-docker - ./src/metacpan-api/wait-for-es.sh http://localhost:9200 elasticsearch_test - name: wait for ES - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test env HARNESS_PERL_SWITCHES="-MDevel::Cover=+ignore,^t/|^test-data/|^etc/" prove -lr --jobs 4 t name: run tests with coverage + command: | + docker compose --profile test run --env HARNESS_PERL_SWITCHES=-MDevel::Cover -v ./cover_db:/app/cover_db/ api-test bash -c 'prove -lr -j4 t && cover -report json' # We are relying on environment variables from the host to be available when # we publish the report, so we publish from the host rather than trying # to propagate env variables to the container. - - run: - command: | - pushd metacpan-docker - docker-compose exec -T api_test cover -report json - name: create coverage report - codecov/upload: - file: metacpan-docker/src/metacpan-api/cover_db/cover.json - - run: - command: | - pushd metacpan-docker - docker-compose logs - docker stats --no-stream - docker ps -a | head - name: docker-compose logs - when: on_fail + file: cover_db/cover.json From 3a2577ddb8d6bfd24d3ca5e9011bed0589baede6 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 10:59:24 +0200 Subject: [PATCH 876/892] update deployed image --- .github/workflows/build-container.yml | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 817561902..12e02ed79 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -34,13 +34,12 @@ jobs: ghcr_username: ${{ github.repository_owner }} ghcr_password: ${{ secrets.GITHUB_TOKEN }} test-target: test -# - name: Update deployed image -# if: steps.find-tag-names.outputs.latest -# uses: metacpan/metacpan-actions/update-deployed-tag:master -# with: -# token: ${{ steps.app-token.outputs.token }} -# app: api -# environment: prod -# base-tag: ${{ steps.find-tag-names.outputs.latest }} -# tag: ${{ steps.find-tag-names.outputs.sha }} - + - name: Update deployed image + if: steps.find-tag-names.outputs.latest + uses: metacpan/metacpan-actions/update-deployed-tag:master + with: + token: ${{ steps.app-token.outputs.token }} + app: api + environment: prod + base-tag: ${{ steps.find-tag-names.outputs.latest }} + tag: ${{ steps.find-tag-names.outputs.sha }} From 7f969fed530d2cc9afa0ff4dfb114d7bbbfcdb97 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 11:34:30 +0200 Subject: [PATCH 877/892] fix critic issue in app.psgi --- app.psgi | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/app.psgi b/app.psgi index 1792a70e0..184c2428c 100644 --- a/app.psgi +++ b/app.psgi @@ -42,7 +42,7 @@ BEGIN { $root_dir ); Log::Log4perl::init($log4perl_config); - package MetaCPAN::Server::WarnHandler; + package MetaCPAN::Server::WarnHandler; ## no critic (Modules::RequireFilenameMatchesPackage) Log::Log4perl->wrapper_register(__PACKAGE__); my $logger = Log::Log4perl->get_logger; $SIG{__WARN__} = sub { $logger->warn(@_) }; From 7c73b13434c0de23e58322fbe38e72802f4cbce6 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 11:34:43 +0200 Subject: [PATCH 878/892] fix excluded files in precious config --- precious.toml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/precious.toml b/precious.toml index 21be953fd..8fdcfe414 100644 --- a/precious.toml +++ b/precious.toml @@ -1,7 +1,9 @@ -excludes = [ - ".build/**", - "blib/**", - "root/assets/**", +exclude = [ + "/.build/**", + "/blib/**", + "/root/assets/**", + "/local/**", + "/test-data/**", ] [commands.perlimports] From d2225020151a961e3ba3804f2fd1af12c5ca90af Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 12:34:56 +0200 Subject: [PATCH 879/892] need all prereqs to run perlimports --- .github/workflows/code-formatting.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml index 5eb31e5e0..647636380 100644 --- a/.github/workflows/code-formatting.yml +++ b/.github/workflows/code-formatting.yml @@ -31,9 +31,6 @@ jobs: cpanfile: 'cpanfile' args: > --resolver=snapshot - --without-runtime - --without-test - --without-build --with-develop - name: Install precious run: ./bin/install-precious /usr/local/bin From ae5bac3f06882dae307d5d491ea3c20a22eed635 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 13:08:32 +0200 Subject: [PATCH 880/892] fix syntax of docker build action --- .github/workflows/build-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 12e02ed79..255651563 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -36,7 +36,7 @@ jobs: test-target: test - name: Update deployed image if: steps.find-tag-names.outputs.latest - uses: metacpan/metacpan-actions/update-deployed-tag:master + uses: metacpan/metacpan-actions/update-deployed-tag@master with: token: ${{ steps.app-token.outputs.token }} app: api From edca14650d398972e054c87c057ebd0a5d1efe3d Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 13:56:02 +0200 Subject: [PATCH 881/892] don't run tests while building docker images The reusable action doesn't support testing via docker compose, and running the tests using just the test target on its own won't work. --- .github/workflows/build-container.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 255651563..41e034507 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -33,7 +33,6 @@ jobs: docker_hub_password: ${{ secrets.DOCKER_HUB_TOKEN }} ghcr_username: ${{ github.repository_owner }} ghcr_password: ${{ secrets.GITHUB_TOKEN }} - test-target: test - name: Update deployed image if: steps.find-tag-names.outputs.latest uses: metacpan/metacpan-actions/update-deployed-tag@master From f36b0dcb173fdb65e5523952e9ceb8af581d79ab Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 17:55:21 +0200 Subject: [PATCH 882/892] fix outputs used for updating k8s --- .github/workflows/build-container.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 41e034507..dbf33c2af 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -40,5 +40,5 @@ jobs: token: ${{ steps.app-token.outputs.token }} app: api environment: prod - base-tag: ${{ steps.find-tag-names.outputs.latest }} - tag: ${{ steps.find-tag-names.outputs.sha }} + base-tag: ${{ fromJSON(steps.build-push.outputs.tag-fq).latest }} + tag: ${{ fromJSON(steps.build-push.outputs.tag-fq).sha }} From 10841ef4dfac0cb77ead26dd31c1db8543bb5417 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 29 Jun 2025 18:02:27 +0200 Subject: [PATCH 883/892] snapshot update: need a configured user even when using signed commits --- .github/workflows/update-snapshot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/update-snapshot.yml b/.github/workflows/update-snapshot.yml index e501ec6f3..60221fb85 100644 --- a/.github/workflows/update-snapshot.yml +++ b/.github/workflows/update-snapshot.yml @@ -15,6 +15,9 @@ jobs: with: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} + - uses: haarg/setup-git-user@v1 + with: + app: ${{ steps.app-token.output.app-slug }} - uses: actions/checkout@v4 with: token: ${{ steps.app-token.outputs.token }} From 5691cb16fa0333ef2ad2ff541503e0fbd476b348 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 30 Jun 2025 10:35:32 +0200 Subject: [PATCH 884/892] gha: another docker build output fix --- .github/workflows/build-container.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index dbf33c2af..4f17ac31f 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -34,7 +34,7 @@ jobs: ghcr_username: ${{ github.repository_owner }} ghcr_password: ${{ secrets.GITHUB_TOKEN }} - name: Update deployed image - if: steps.find-tag-names.outputs.latest + if: ${{ fromJSON(steps.build-push.outputs.tag-fq).latest }} uses: metacpan/metacpan-actions/update-deployed-tag@master with: token: ${{ steps.app-token.outputs.token }} From 4cb01be908a4b4151a05c2759f5d02fa28063ccf Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 13 Jul 2025 23:23:54 +0200 Subject: [PATCH 885/892] add MetaCPAN::Util::to_bool This simplifies converting to a JSON boolean. --- lib/MetaCPAN/Util.pm | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/lib/MetaCPAN/Util.pm b/lib/MetaCPAN/Util.pm index 77b0855ae..582af0637 100644 --- a/lib/MetaCPAN/Util.pm +++ b/lib/MetaCPAN/Util.pm @@ -37,6 +37,7 @@ use Sub::Exporter -setup => { true false is_bool + to_bool MAX_RESULT_WINDOW ) ] }; @@ -44,9 +45,13 @@ use Sub::Exporter -setup => { # Limit the maximum result window to 1000, really should be enough! use constant MAX_RESULT_WINDOW => 1000; -*true = \&Cpanel::JSON::XS::true; -*false = \&Cpanel::JSON::XS::false; +sub true (); +*true = \&Cpanel::JSON::XS::true; +sub false (); +*false = \&Cpanel::JSON::XS::false; +sub is_bool ($); *is_bool = \&Cpanel::JSON::XS::is_bool; +sub to_bool ($) { $_[0] ? true : false } sub root_dir { Cwd::abs_path( File::Spec->catdir( From 3d68cb3811b180c5237da2bd98a91ab08ad5776e Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 13 Jul 2025 23:24:42 +0200 Subject: [PATCH 886/892] match files to provides data based on exact match, not suffix The files listed in provides data should be an exact match for the file path we are using. The regex matching was added in f3543c11ea543d8b9a688cfbc4c189b653c4b147 saying only "fixed regression". I can barely guess at what it was supposedly fixing, but it was definitely not the correct fix. --- lib/MetaCPAN/Model/Release.pm | 9 +++++---- t/lib/MetaCPAN/Tests/Release.pm | 16 ++++++---------- t/release/file-duplicates.t | 9 +-------- 3 files changed, 12 insertions(+), 22 deletions(-) diff --git a/lib/MetaCPAN/Model/Release.pm b/lib/MetaCPAN/Model/Release.pm index 83ef2f1d6..b756770dc 100644 --- a/lib/MetaCPAN/Model/Release.pm +++ b/lib/MetaCPAN/Model/Release.pm @@ -481,15 +481,16 @@ sub _modules_from_meta { my $provides = $self->metadata->provides; my $files = $self->files; + my %files = map +( $_->path => $_ ), @$files; foreach my $module ( sort keys %$provides ) { my $data = $provides->{$module}; my $path = File::Spec->canonpath( $data->{file} ); - # Obey no_index and take the shortest path if multiple files match. - my ($file) = sort { length( $a->path ) <=> length( $b->path ) } - grep { $_->indexed && $_->path =~ /\Q$path\E$/ } @$files; + my $file = $files{$path} + or next; + + next unless $file->indexed; - next unless $file; $file->add_module( { name => $module, version => $data->{version}, diff --git a/t/lib/MetaCPAN/Tests/Release.pm b/t/lib/MetaCPAN/Tests/Release.pm index 1da09c204..8f7c7a6c7 100644 --- a/t/lib/MetaCPAN/Tests/Release.pm +++ b/t/lib/MetaCPAN/Tests/Release.pm @@ -230,18 +230,14 @@ test 'modules in Packages-1.103' => sub { = map { ( $_->{path} => $_->{module} ) } @{ $self->module_files }; foreach my $path ( sort keys %{ $self->modules } ) { - my $desc = "File '$path' has expected modules"; - if ( my $got = delete $module_files{$path} ) { - my $got = [ map +{%$_}, @$got ]; - $_->{associated_pod} //= undef for @$got; + my $desc = "File '$path' has expected modules"; + my $got_modules = delete $module_files{$path} || []; + my $got = [ map +{%$_}, @$got_modules ]; + $_->{associated_pod} //= undef for @$got; # We may need to sort modules by name, I'm not sure if order is reliable. - is_deeply $got, $self->modules->{$path}, $desc - or diag Test::More::explain($got); - } - else { - ok( 0, $desc ); - } + is_deeply $got, $self->modules->{$path}, $desc + or diag Test::More::explain($got); } is( scalar keys %module_files, 0, 'all module files tested' ) diff --git a/t/release/file-duplicates.t b/t/release/file-duplicates.t index 226bc5df4..50ca19b88 100644 --- a/t/release/file-duplicates.t +++ b/t/release/file-duplicates.t @@ -28,14 +28,7 @@ test_release( indexed => true, associated_pod => undef, } ], - 'lib/Dupe.pm' => [ { - name => 'Dupe', - version => '0.993', - version_numified => '0.993', - authorized => true, - indexed => false, - associated_pod => undef, - } ], + 'lib/Dupe.pm' => [], 'DupeX/Dupe.pm' => [ { name => 'DupeX::Dupe', From 08bd7ef9c76092eaa202fab08717fe72c8883b8d Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Sun, 13 Jul 2025 23:28:11 +0200 Subject: [PATCH 887/892] don't scan module content ourselves to detect packages Module::Metadata, Parse::PMFile, or provides metadata already tell us what modules exist in a file. They will not include packages hidden using the "hide from PAUSE" trick. This was possibly needed based on other metadata mishandling. Some of the tests were relying on this behavior and have been removed. They were expecting the data provided in the test to be overridden by the additional package check, but the module list would never have been given to them in our indexing code. --- lib/MetaCPAN/Document/File.pm | 6 +--- lib/MetaCPAN/Document/Module.pm | 25 --------------- t/document/file.t | 17 ++-------- t/document/module.t | 55 --------------------------------- 4 files changed, 4 insertions(+), 99 deletions(-) diff --git a/lib/MetaCPAN/Document/File.pm b/lib/MetaCPAN/Document/File.pm index 97028fce2..c9fd79c8a 100644 --- a/lib/MetaCPAN/Document/File.pm +++ b/lib/MetaCPAN/Document/File.pm @@ -917,11 +917,7 @@ sub set_indexed { next; } - $mod->_set_indexed( - $mod->hide_from_pause( ${ $self->content }, $self->name ) - ? false - : true - ); + $mod->_set_indexed(true); } if ( my $doc_name = $self->documentation ) { diff --git a/lib/MetaCPAN/Document/Module.pm b/lib/MetaCPAN/Document/Module.pm index 3f1fc5358..a3901bae1 100644 --- a/lib/MetaCPAN/Document/Module.pm +++ b/lib/MetaCPAN/Document/Module.pm @@ -111,31 +111,6 @@ sub _build_version_numified { my $bom = qr/(?:\x00\x00\xfe\xff|\xff\xfe\x00\x00|\xfe\xff|\xff\xfe|\xef\xbb\xbf)/; -sub hide_from_pause { - my ( $self, $content, $file_name ) = @_; - return 0 if defined($file_name) && $file_name =~ m{\.pm\.PL\z}; - my $pkg = $self->name; - my $pkg_match = join q[(?:::|')], map quotemeta, split m{::}, $pkg; - -# This regexp is *almost* the same as $PKG_REGEXP in Module::Metadata. -# [b] We need to allow/ignore a possible BOM since we read in binary mode. -# Module::Metadata, for example, checks for a BOM and then sets the encoding. -# [s] We change `\s` to `\h` because we want to verify that it's on one line. -# [p] We replace $PKG_NAME_REGEXP with the specific package we're looking for. -# [v] Simplify the optional whitespace/version group ($V_NUM_REGEXP). - return $content =~ / # match a package declaration - ^ # start of line - (?:\A$bom)? # possible BOM at the start of the file [b] - [\h\{;]* # intro chars on a line [s] - package # the word 'package' - \h+ # whitespace [s] - ($pkg_match) # the package name [p] - (\h+ v?[0-9._]+)? # optional version number (preceded by whitespace) [v] - \h* # optional whitesapce [s] - [;\{] # semicolon line terminator or block start - /mx ? 0 : 1; -} - =head2 set_associated_pod Expects an instance C<$file> of L as first parameter diff --git a/t/document/file.t b/t/document/file.t index 6026b979f..0da986ebb 100644 --- a/t/document/file.t +++ b/t/document/file.t @@ -227,11 +227,6 @@ END is( $file->abstract, 'An object containing information about how to get access to teh Moby databases, resources, etc. from the mobycentral.config file' ); - is( - $file->module->[0] - ->hide_from_pause( ${ $file->content }, $file->name ), - 0, 'indexed' - ); is( $file->documentation, 'MOBY::Config' ); is( $file->level, 2 ); test_attributes $file, { @@ -302,13 +297,9 @@ AS-specific methods for Number::Phone 1; END - my $file = new_file_doc( - module => [ { name => 'Number::Phone::NANP::ASS', version => 1.1 } ], - content => \$content, - ); - is( $file->sloc, 8, '8 lines of code' ); - is( $file->slop, 4, '4 lines of pod' ); - is( $file->module->[0]->hide_from_pause($content), 1, 'not indexed' ); + my $file = new_file_doc( content => \$content, ); + is( $file->sloc, 8, '8 lines of code' ); + is( $file->slop, 4, '4 lines of pod' ); is( $file->abstract, 'AS-specific methods for Number::Phone', @@ -322,8 +313,6 @@ END is( $file->documentation, 'Number::Phone::NANP::AS', 'document text' ); is_deeply( $file->pod_lines, [ [ 18, 7 ] ], 'correct pod_lines' ); - is( $file->module->[0]->version_numified, - 1.1, 'numified version has been calculated' ); is( ${ $file->pod }, diff --git a/t/document/module.t b/t/document/module.t index d21fcac7d..c2c2400b4 100644 --- a/t/document/module.t +++ b/t/document/module.t @@ -5,61 +5,6 @@ use lib 't/lib'; use MetaCPAN::Document::Module (); use Test::More; -subtest hide_from_pause => sub { - foreach my $test ( - - # The original: - [ 'No::CommentNL' => "package # hide\n No::CommentNL;" ], - - # I'm not sure how PAUSE handles this one but currently we ignore it. - [ 'No::JustNL' => "package \n No::JustNL;" ], - - # The good ones: - [ 'Pkg' => 'package Pkg;' ], - [ 'Pkg::Ver' => 'package Pkg::Ver v1.2.3;' ], - [ 'Pkg::Block' => 'package Pkg::Block { our $var = 1 }' ], - [ - 'Pkg::VerBlock' => 'package Pkg::VerBlock 1.203 { our $var = 1 }' - ], - [ 'Pkg::SemiColons' => '; package Pkg::SemiColons; $var' ], - [ 'Pkg::InABlock' => '{ package Pkg::InABlock; $var }' ], - - # This doesn't work as a BOM can only appear at the start of a file. - #[ 'Pkg::AfterABOM' => "\xef\xbb\xbfpackage Pkg::AfterABOM" ], - - [ 'No::JustVar' => qq["\n\$package No::JustVar;\n"] ], - - # This shouldn't match, but there's only so much we can do... - # we're not going to eval the whole file to figure it out. - [ 'Pkg::InsideStr' => qq["\n package Pkg::InsideStr;\n"] ], - - [ 'No::Comment' => qq[# package No::Comment;\n] ], - [ 'No::Different' => q[package No::Different::Pkg;] ], - [ 'No::PkgWithNum' => qq["\npackage No::PkgWithNumv2.3;\n"] ], - [ 'No::CrazyChars' => qq["\npackage No::CrazyChars\[0\];\n"] ], - ) - { - my ( $name, $content ) = @$test; - - subtest $name => sub { - my $module = MetaCPAN::Document::Module->new( name => $name ); - - SKIP: { - skip( 'Perl 5.14 needed for package block compilation', 1 ) - if $] < 5.014; - ## no critic - ok eval "sub { no strict; $content }", "code compiles" - or diag $@; - } - - my ($hidden) = ( $name =~ /^No::/ ? 1 : 0 ); - - is $module->hide_from_pause($content), $hidden, - "hide_from_pause is $hidden"; - }; - } -}; - subtest set_associated_pod => sub { test_associated_pod( 'Squirrel', [qw( lib/Squirrel.pod )], 'lib/Squirrel.pod' ); From 67adff674d3de6c5d5c5e91543e322cbe0e07cb0 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 14 Jul 2025 00:11:36 +0200 Subject: [PATCH 888/892] generate code coverage in a format compatible with Codecov Codecov doesn't understand the JSON format generated by Devel::Cover. Use the Codecovbash report plugin to generate a suitable file. --- .circleci/config.yml | 4 ++-- cpanfile | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/.circleci/config.yml b/.circleci/config.yml index 0aa915976..9a883d851 100644 --- a/.circleci/config.yml +++ b/.circleci/config.yml @@ -30,9 +30,9 @@ jobs: - run: name: run tests with coverage command: | - docker compose --profile test run --env HARNESS_PERL_SWITCHES=-MDevel::Cover -v ./cover_db:/app/cover_db/ api-test bash -c 'prove -lr -j4 t && cover -report json' + docker compose --profile test run --env HARNESS_PERL_SWITCHES=-MDevel::Cover -v ./cover_db:/app/cover_db/ api-test bash -c 'prove -lr -j4 t && cover -report codecovbash' # We are relying on environment variables from the host to be available when # we publish the report, so we publish from the host rather than trying # to propagate env variables to the container. - codecov/upload: - file: cover_db/cover.json + file: cover_db/codecov.json diff --git a/cpanfile b/cpanfile index e647fcde6..dd1d4513f 100644 --- a/cpanfile +++ b/cpanfile @@ -162,4 +162,5 @@ on develop => sub { requires 'PPIx::Regexp', '0.085'; # Perl::Critic requires 'String::Format', '1.18'; # Perl::Critic requires 'Devel::Cover'; + requires 'Devel::Cover::Report::Codecovbash'; }; From ac26b1923dd9f9a8ac28c8f791f7fa79f497c6b2 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Wed, 16 Jul 2025 21:09:31 +0200 Subject: [PATCH 889/892] better check for previously extracted source files When extracting source files for the API server, we always fix the extracted path have a single directory with the same name as the release. Since we are also using the directory as a staging area, we end up with the release name doubled. When checking for a previously extracted tarball, use the full path of the directory, since old versions of the software or incomplete extractions may have left the parent directory in place but not have the full proper new paths. --- lib/MetaCPAN/Server/Model/Source.pm | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Model/Source.pm b/lib/MetaCPAN/Server/Model/Source.pm index 954a0ce43..9c40461db 100644 --- a/lib/MetaCPAN/Server/Model/Source.pm +++ b/lib/MetaCPAN/Server/Model/Source.pm @@ -88,7 +88,7 @@ sub path { return $source if -e $source; return undef - if -e $source_base; # previously extracted, but file does not exist + if -e $source_base->child($distvname); # previously extracted, but file does not exist my $release_data = $self->es_query->release->by_author_and_name( $pauseid, $distvname ) From 8033be9377a6b5906e77d5c5bdae637cddfe0359 Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 11 Aug 2025 20:34:44 +0200 Subject: [PATCH 890/892] better comment for source 404 --- lib/MetaCPAN/Server/Model/Source.pm | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/lib/MetaCPAN/Server/Model/Source.pm b/lib/MetaCPAN/Server/Model/Source.pm index 9c40461db..8ff322713 100644 --- a/lib/MetaCPAN/Server/Model/Source.pm +++ b/lib/MetaCPAN/Server/Model/Source.pm @@ -87,8 +87,11 @@ sub path { my $source = $source_base->child( $distvname, @file ); return $source if -e $source; + + # if the directory exists, we already extracted the archive, so if the + # file didn't exist, we can stop here return undef - if -e $source_base->child($distvname); # previously extracted, but file does not exist + if -e $source_base->child($distvname); my $release_data = $self->es_query->release->by_author_and_name( $pauseid, $distvname ) From fdc134b65f2359e04ba59ba88d6c068de304879a Mon Sep 17 00:00:00 2001 From: Graham Knop Date: Mon, 11 Aug 2025 20:41:21 +0200 Subject: [PATCH 891/892] fix the pod end point when given a module name The find_pod method would sometimes return the source and sometimes a full record from Elasticsearch. Fix the return to always return the source, and adjust the caller to work with this form. --- lib/MetaCPAN/Query/File.pm | 2 +- lib/MetaCPAN/Server/Controller/Pod.pm | 3 +-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/lib/MetaCPAN/Query/File.pm b/lib/MetaCPAN/Query/File.pm index 2280b2ba7..15696c491 100644 --- a/lib/MetaCPAN/Query/File.pm +++ b/lib/MetaCPAN/Query/File.pm @@ -679,7 +679,7 @@ sub find_pod { query => $query, }, ); - return $pod_file->{hits}{hits}[0]; + return $pod_file->{hits}{hits}[0]->{_source}; } else { return $file; diff --git a/lib/MetaCPAN/Server/Controller/Pod.pm b/lib/MetaCPAN/Server/Controller/Pod.pm index 9f2edc84b..d7daf60d7 100644 --- a/lib/MetaCPAN/Server/Controller/Pod.pm +++ b/lib/MetaCPAN/Server/Controller/Pod.pm @@ -38,8 +38,7 @@ sub get : Path('') : Args(1) { my ( $self, $c, $module ) = @_; $module = $c->model('ESQuery')->file->find_pod($module) or $c->detach( '/not_found', [] ); - $c->forward( 'find', - [ map { $module->{_source}{$_} } qw(author release path) ] ); + $c->forward( 'find', [ map { $module->{$_} } qw(author release path) ] ); } sub find_dist_links { From 8a66698418b5093cb369def53c23ad31a1684f7a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Aug 2025 02:11:52 +0000 Subject: [PATCH 892/892] Bump actions/checkout from 4 to 5 Bumps [actions/checkout](https://github.com/actions/checkout) from 4 to 5. - [Release notes](https://github.com/actions/checkout/releases) - [Changelog](https://github.com/actions/checkout/blob/main/CHANGELOG.md) - [Commits](https://github.com/actions/checkout/compare/v4...v5) --- updated-dependencies: - dependency-name: actions/checkout dependency-version: '5' dependency-type: direct:production update-type: version-update:semver-major ... Signed-off-by: dependabot[bot] --- .github/workflows/build-container.yml | 2 +- .github/workflows/code-formatting.yml | 2 +- .github/workflows/update-snapshot.yml | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-container.yml b/.github/workflows/build-container.yml index 4f17ac31f..00d325bee 100644 --- a/.github/workflows/build-container.yml +++ b/.github/workflows/build-container.yml @@ -23,7 +23,7 @@ jobs: app-id: ${{ secrets.APP_ID }} private-key: ${{ secrets.APP_PRIVATE_KEY }} owner: metacpan - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: token: ${{ steps.app-token.outputs.token }} - uses: metacpan/metacpan-actions/docker-build-push@master diff --git a/.github/workflows/code-formatting.yml b/.github/workflows/code-formatting.yml index 647636380..b833d372c 100644 --- a/.github/workflows/code-formatting.yml +++ b/.github/workflows/code-formatting.yml @@ -15,7 +15,7 @@ jobs: runs-on: ubuntu-24.04 name: Code Formatting steps: - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: fetch-depth: 0 - name: Fetch base ref diff --git a/.github/workflows/update-snapshot.yml b/.github/workflows/update-snapshot.yml index 60221fb85..816301081 100644 --- a/.github/workflows/update-snapshot.yml +++ b/.github/workflows/update-snapshot.yml @@ -18,7 +18,7 @@ jobs: - uses: haarg/setup-git-user@v1 with: app: ${{ steps.app-token.output.app-slug }} - - uses: actions/checkout@v4 + - uses: actions/checkout@v5 with: token: ${{ steps.app-token.outputs.token }} - name: Update cpanfile.snapshot