[commit: http://hg.dwscoalition.org/dw-free/rev/99ffce26d485]

http://bugs.dwscoalition.org/show_bug.cgi?id=2391

Adds the ability to import communities from a remote site to a local
community. This is a pretty straightforward system on top of the existing
importer and follows many of the same rules.

Some notes:

* There is a configuration option to turn this on, as well as a user cap
that needs to be present.

* You can disable importing comments for accounts with more than X comments,
if you want to help with load.

* This requires that you be a maintainer of the community on the local site
and the remote site. (We don't want people arbitrarily importing communities
they don't own.)

Patch by mark.

Files modified:

• bin/checkconfig.pl
• cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm
• cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm
• cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm
• cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm
• cgi-bin/LJ/Protocol.pm
• cgi-bin/LJ/Talk.pm
• cgi-bin/LJ/User.pm
• cgi-bin/LJ/Widget/ImportChooseData.pm
• cgi-bin/LJ/Widget/ImportChooseSource.pm
• cgi-bin/LJ/Widget/ImportStatus.pm
• cvs/multicvs.conf
• doc/config-local.pl.txt
• htdocs/stc/importer.css
• htdocs/tools/importer.bml
• htdocs/tools/importer.bml.text

--------------------------------------------------------------------------------
diff -r e901769c10a1 -r 99ffce26d485 bin/checkconfig.pl
--- a/bin/checkconfig.pl	Sat Dec 24 16:23:51 2011 +0800
+++ b/bin/checkconfig.pl	Mon Dec 26 07:56:11 2011 +0000
@@ -237,6 +237,7 @@
                     deb => 'libio-aoi-perl',
                     opt => 'Required for Perlbal',
                  },
+               "LWPx::ParanoidAgent" => { deb => 'liblwpx-paranoidagent-perl' },
               );
 
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -221,6 +221,12 @@
             }
         );
         $parser->parse( $content );
+
+        # this is the best place to test for too many comments. if this site is limiting
+        # the comment imports for some reason or another, we can bail here.
+        return $fail->( $LJ::COMMENT_IMPORT_ERROR || 'Too many comments to import.' )
+            if defined $LJ::COMMENT_IMPORT_MAX && defined $server_max_id &&
+                $server_max_id > $LJ::COMMENT_IMPORT_MAX;
     }
     $log->( 'Finished fetching metadata.' );
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -76,7 +76,7 @@
         q{UPDATE import_items SET status = 'ready'
           WHERE userid = ? AND item IN ('lj_friends', 'lj_entries')
           AND import_data_id = ? AND status = 'init'},
-        undef, $u->id, $opts->{import_data_id}        
+        undef, $u->id, $opts->{import_data_id}
     );
 
     return $ok->();
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -53,6 +53,9 @@
         or return $fail->( 'Unable to load target with id %d.', $data->{userid} );
     $0 = sprintf( 'content-importer [tags: %s(%d)]', $u->user, $u->id );
 
+    my $dbh = LJ::get_db_writer()
+        or return $temp_fail->( 'Unable to get global master database handle' );
+
     # get tags
     my $r = $class->call_xmlrpc( $data, 'getusertags' );
     return $temp_fail->( 'XMLRPC failure: ' . $r->{faultString} )
@@ -60,6 +63,16 @@
 
     DW::Worker::ContentImporter::Local::Tags->merge_tags( $u, $r->{tags} );
 
+    # if this is a community, it is now our job to schedule the entry import
+    if ( $u->is_community ) {
+        $dbh->do(
+            q{UPDATE import_items SET status = 'ready'
+              WHERE userid = ? AND item = 'lj_entries'
+                  AND import_data_id = ? AND status = 'init'},
+            undef, $u->id, $opts->{import_data_id}
+        );
+    }
+
     return $ok->();
 }
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -77,6 +77,19 @@
     return $temp_fail->( 'XMLRPC failure: ' . $r->{faultString} )
         if ! $r || $r->{fault};
 
+    # If this is a community import, we have to do a second step now to make sure
+    # that the user is an administrator of the remote community. The best way I can
+    # come up with is try to unban the owner -- if it works, you're an admin.
+    if ( $data->{usejournal} ) {
+        $r = $class->call_xmlrpc( $data, 'consolecommand', {
+            commands => [ "ban_unset $data->{username} from $data->{usejournal}" ],
+        } );
+        return $temp_fail->( 'XMLRPC failure: ' . $r->{faultString} )
+            if ! $r || $r->{fault};
+        return $fail->( 'You are not an administrator/maintainer of the remote community.' )
+            unless $r->{results}->[0]->{output}->[0]->[0] eq 'success';
+    }
+
     # mark the next group as ready to schedule
     my $dbh = LJ::get_db_writer();
     $dbh->do(
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Protocol.pm
--- a/cgi-bin/LJ/Protocol.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Protocol.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -1361,6 +1361,14 @@
     unless ($req->{'props'}->{"opt_backdated"}) {
         $rlogtime -= $now;
     }
+    my $logtime = "FROM_UNIXTIME($now)";
+
+    # this is when the entry was posted. for most cases this is accurate but in case
+    # we're using the importer in the community case, it will mess life up.
+    if ( $importer_bypass && $posterid != $ownerid ) {
+        $logtime = $qeventtime;
+        $rlogtime = "$LJ::EndOfTime - UNIX_TIMESTAMP($qeventtime)";
+    }
 
     my $dupsig = Digest::MD5::md5_hex(join('', map { $req->{$_} }
                                            qw(subject event usejournal security allowmask)));
@@ -1523,7 +1531,7 @@
     my $dberr;
     $uowner->log2_do(\$dberr, "INSERT INTO log2 (journalid, jitemid, posterid, eventtime, logtime, security, ".
                      "allowmask, replycount, year, month, day, revttime, rlogtime, anum) ".
-                     "VALUES ($ownerid, $jitemid, $posterid, $qeventtime, FROM_UNIXTIME($now), $qsecurity, $qallowmask, ".
+                     "VALUES ($ownerid, $jitemid, $posterid, $qeventtime, $logtime, $qsecurity, $qallowmask, ".
                      "0, $req->{'year'}, $req->{'mon'}, $req->{'day'}, $LJ::EndOfTime-".
                      "UNIX_TIMESTAMP($qeventtime), $rlogtime, $anum)");
     return $fail->($err,501,$dberr) if $dberr;
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Talk.pm
--- a/cgi-bin/LJ/Talk.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Talk.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -1300,7 +1300,8 @@
                 }
                 $post->{picid} = $id;
                 $post->{pickw} = $kw;
-                push @load_pic, [ $pu, $id ];
+                push @load_pic, [ $pu, $id ]
+                    if defined $id;
             }
             load_userpics( $opts->{userpicref}, \@load_pic );
         }
@@ -1336,7 +1337,7 @@
     foreach my $row (@{$idlist})
     {
         my ($u, $id) = @$row;
-        next unless ref $u;
+        next unless ref $u && defined $id;
 
         if ($LJ::CACHE_USERPIC{$id}) {
             $upics->{$id} = $LJ::CACHE_USERPIC{$id};
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/User.pm
--- a/cgi-bin/LJ/User.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/User.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -2010,6 +2010,10 @@
     return $_[0]->get_cap( 'disable_can_post' ) ? 1 : 0;
 }
 
+sub can_import_comm {
+    return $_[0]->get_cap( 'import_comm' ) ? 1 : 0;
+}
+
 sub can_receive_vgifts_from {
     my ( $u, $remote, $is_anon ) = @_;
     $remote ||= LJ::get_remote();
@@ -3193,7 +3197,6 @@
 
     my ($url, $name);
     if ($id->typeid eq 'O') {
-        require Net::OpenID::Consumer;
         $url = $id->value;
         $name = Net::OpenID::VerifiedIdentity::DisplayOfURL($url, $LJ::IS_DEV_SERVER);
         $name = LJ::Hooks::run_hook("identity_display_name", $name) || $name;
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Widget/ImportChooseData.pm
--- a/cgi-bin/LJ/Widget/ImportChooseData.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Widget/ImportChooseData.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -34,12 +34,28 @@
             display_name => $class->ml( 'widget.importstatus.item.lj_bio' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_bio.desc' ),
             selected => 0,
+            comm_okay => 1,
+        },
+        {
+            name => 'lj_friends',
+            display_name => $class->ml( 'widget.importstatus.item.lj_friends' ),
+            desc => $class->ml( 'widget.importchoosedata.item.lj_friends.desc' ),
+            selected => 0,
+            comm_okay => 0,
+        },
+        {
+            name => 'lj_friendgroups',
+            display_name => $class->ml( 'widget.importstatus.item.lj_friendgroups' ),
+            desc => $class->ml( 'widget.importchoosedata.item.lj_friendgroups.desc', { sitename => $LJ::SITENAMESHORT } ),
+            selected => 0,
+            comm_okay => 0,
         },
         {
             name => 'lj_entries',
             display_name => $class->ml( 'widget.importstatus.item.lj_entries' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_entries.desc' ),
             selected => 0,
+            comm_okay => 1,
 
             suboptions => [
                 {
@@ -51,34 +67,25 @@
             ]
         },
         {
-            name => 'lj_friends',
-            display_name => $class->ml( 'widget.importstatus.item.lj_friends' ),
-            desc => $class->ml( 'widget.importchoosedata.item.lj_friends.desc' ),
-            selected => 0,
-        },
-        {
             name => 'lj_comments',
             display_name => $class->ml( 'widget.importstatus.item.lj_comments' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_comments.desc' ),
             selected => 0,
+            comm_okay => 1,
         },
         {
             name => 'lj_tags',
             display_name => $class->ml( 'widget.importstatus.item.lj_tags' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_tags.desc' ),
             selected => 0,
+            comm_okay => 1,
         },
         {
             name => 'lj_userpics',
             display_name => $class->ml( 'widget.importstatus.item.lj_userpics' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_userpics.desc', { sitename => $LJ::SITENAMESHORT } ),
             selected => 0,
-        },
-        {
-            name => 'lj_friendgroups',
-            display_name => $class->ml( 'widget.importstatus.item.lj_friendgroups' ),
-            desc => $class->ml( 'widget.importchoosedata.item.lj_friendgroups.desc', { sitename => $LJ::SITENAMESHORT } ),
-            selected => 0,
+            comm_okay => 1,
         },
     );
 
@@ -90,6 +97,8 @@
     $ret .= "<div class='importoptions'>";
 
     foreach my $option ( @options ) {
+        next unless $u->is_person || $option->{comm_okay};
+
         $ret .= "<div class='importoption'>";
         $ret .= $class->html_check(
             name => $option->{name},
@@ -166,6 +175,12 @@
     # everybody needs a verifier
     $post->{lj_verify} = 1;
 
+    # and finally, make sure modes that make no sense for communities are off
+    if ( $u->is_community ) {
+        $post->{lj_friends} = 0;
+        $post->{lj_friendgroups} = 0;
+    }
+
     return ( ret => LJ::Widget::ImportConfirm->render( %$post ) );
 }
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Widget/ImportChooseSource.pm
--- a/cgi-bin/LJ/Widget/ImportChooseSource.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Widget/ImportChooseSource.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -88,7 +88,7 @@
     $ret .= "<label for='password'>" . $class->ml( 'widget.importchoosesource.password' ) . "</label>";
     $ret .= $class->html_text( type => 'password', name => 'password' );
     $ret .= "</div>";
-    if ( $LJ::ALLOW_COMM_IMPORT{$u->user} ) {
+    if ( $u->is_community ) {
         $ret .= "<div class='i-usejournal'>";
         $ret .= "<label for='usejournal'>" . $class->ml( 'widget.importchoosesource.usejournal' ) . "</label>";
         $ret .= $class->html_text( name => 'usejournal', maxlength => 255 );
@@ -117,13 +117,13 @@
     $un =~ s/-/_/g;
 
     # be sure to sanitize the usejournal
-    my $uj = undef;
-    if ( $LJ::ALLOW_COMM_IMPORT{$u->user} ) {
+    my $uj;
+    if ( $u->is_community ) {
         $uj = LJ::trim( lc $post->{usejournal} );
         $uj =~ s/-/_/g;
     }
 
-    my $pw = $post->{password};
+    my $pw = LJ::trim( $post->{password} );
     return ( ret => $class->ml( 'widget.importchoosesource.error.nocredentials' ) )
         unless $un && $pw;
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Widget/ImportStatus.pm
--- a/cgi-bin/LJ/Widget/ImportStatus.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Widget/ImportStatus.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -42,7 +42,7 @@
             my $import_item = $items->{$importid};
 
             $ret .= "<tr><td colspan='4' class='table-header'>" . $class->ml( 'widget.importstatus.whichaccount', { user => $import_item->{user}, host => $import_item->{host} } ) . " | ";
-            $ret .= "<a href='$LJ::SITEROOT/tools/importer'>" . $class->ml( 'widget.importstatus.refresh' ) . "</a></td></tr>";
+            $ret .= "<a href='$LJ::SITEROOT/tools/importer?authas=" . $u->user . "'>" . $class->ml( 'widget.importstatus.refresh' ) . "</a></td></tr>";
             foreach my $item ( sort keys %{$import_item->{items}} ) {
                 my $i = $import_item->{items}->{$item};
                 my $color = { init => '#333333', ready => '#3333aa', queued => '#33aa33', failed => '#aa3333', succeeded => '#00ff00' }->{$i->{status}};
diff -r e901769c10a1 -r 99ffce26d485 cvs/multicvs.conf
--- a/cvs/multicvs.conf	Sat Dec 24 16:23:51 2011 +0800
+++ b/cvs/multicvs.conf	Mon Dec 26 07:56:11 2011 +0000
@@ -23,7 +23,6 @@
 SVN(memcached)            = http://code.livejournal.org/svn/memcached/trunk/
 SVN(CSS-Cleaner)          = http://code.livejournal.org/svn/CSS-Cleaner/trunk/
 SVN(Sys-Syscall)          = http://code.livejournal.org/svn/Sys-Syscall/trunk/
-SVN(LWPx-ParanoidAgent)   = http://code.livejournal.org/svn/LWPx-ParanoidAgent/trunk/
 SVN(Danga-Socket)         = http://code.livejournal.org/svn/Danga-Socket/trunk/
 SVN(mogilefs)             = http://code.sixapart.com/svn/mogilefs/trunk/ @1459
 SVN(TheSchwartz)          = http://code.sixapart.com/svn/TheSchwartz/trunk/
@@ -46,7 +45,6 @@
 TheSchwartz-Worker-SendEmail/lib              cgi-bin/
 Sys-Syscall/lib                               cgi-bin
 Danga-Socket/lib/Danga/Socket.pm              cgi-bin/Danga/Socket.pm
-LWPx-ParanoidAgent/lib/LWPx                   cgi-bin/LWPx
 #openid/perl/Net-OpenID-Consumer/lib           cgi-bin
 #openid/perl/Net-OpenID-Server/lib             cgi-bin
 #openid/perl/Net-OpenID-Common/lib             cgi-bin
diff -r e901769c10a1 -r 99ffce26d485 doc/config-local.pl.txt
--- a/doc/config-local.pl.txt	Sat Dec 24 16:23:51 2011 +0800
+++ b/doc/config-local.pl.txt	Mon Dec 26 07:56:11 2011 +0000
@@ -113,6 +113,14 @@
     #    rename => [ 15, undef, undef, 150 ],
     #);
 
+    # You can turn on/off community importing here.
+    $ALLOW_COMM_IMPORTS = 0;
+
+    # If this is defined and a number, if someone tries to import more than this many
+    # comments in a single import, the error specified will be raised and the job will fail.
+    $COMMENT_IMPORT_MAX = undef;
+    $COMMENT_IMPORT_ERROR = "Importing more than 10,000 comments is currently disabled.";
+
 }
 
 1;
diff -r e901769c10a1 -r 99ffce26d485 htdocs/stc/importer.css
--- a/htdocs/stc/importer.css	Sat Dec 24 16:23:51 2011 +0800
+++ b/htdocs/stc/importer.css	Mon Dec 26 07:56:11 2011 +0000
@@ -68,6 +68,32 @@
     margin-left: -10px;
 }
 
+.importer .usejournal {
+    border-top: 1px solid #CCC;
+    padding: 10px;
+    margin-bottom: 10px;
+}
+
+.importer .usejournal select {
+    position: absolute;
+    left: 22em;
+    margin-top: -1px;
+}
+
+.importer .usejournal input {
+    position: absolute;
+    left: 22em;
+    margin-top: -1px;
+}
+
+.importer .usejournal label {
+    font-weight: bold;
+}
+
+.importer .usejournal div {
+    margin-bottom: 5px;
+}
+
 .importer .credentials {
     border-top: 1px solid #CCC;
     padding: 10px;
@@ -97,7 +123,7 @@
 }
 
 .importer .importantnote {
-    color: #666;
+    color: #667;
     font-size: 0.9em;
     padding: 10px;
     border-top: 1px solid #CCC;
diff -r e901769c10a1 -r 99ffce26d485 htdocs/tools/importer.bml
--- a/htdocs/tools/importer.bml	Sat Dec 24 16:23:51 2011 +0800
+++ b/htdocs/tools/importer.bml	Mon Dec 26 07:56:11 2011 +0000
@@ -32,11 +32,19 @@
 
     $title = $ML{'.title'};
 
-    my $remote = LJ::get_remote()
-        or return "<?needlogin?>";
+    my $u = LJ::get_remote();
+    my $remote = LJ::get_effective_remote();
+    return "<?needlogin?>" unless $u && $remote;
+
+    # if these aren't the same users, make sure we're allowed to do a community import
+    unless ( $u->equals( $remote ) ) {
+        return $ML{'.error.cant_import_comm'}
+            unless $LJ::ALLOW_COMM_IMPORTS &&
+                ( $u->can_import_comm || $remote->can_import_comm );
+    }
 
     return $ML{'.error.notperson'}
-        unless $remote->is_person;
+        unless $u->is_person;
 
     if ( LJ::did_post() ) {
         return "<?h1 $ML{'Error'} h1?><?p $ML{'error.invalidform'} p?>"
@@ -46,13 +54,22 @@
         if ( $from_post{notloggedin} ) {
             return "<?needlogin?>";
         } elsif ( $from_post{refresh} ) {
-            return BML::redirect( "$LJ::SITEROOT/tools/importer" );
+            my $authas = $remote->equals( $u ) ? "" : "?authas=" . $remote->user;
+            return BML::redirect( "$LJ::SITEROOT/tools/importer$authas" );
         } elsif ( $from_post{ret} ) {
             return "<div class='importer'>$from_post{ret}</div>";
         }
     }
 
-    my $ret = "<p class='intro'>" . BML::ml( '.intro', { sitename => $LJ::SITENAMESHORT, user => $remote->ljuser_display } ) . "</p>";
+    # user switcher
+    my $ret = '';
+    if ( $LJ::ALLOW_COMM_IMPORTS && $u->can_import_comm ) {
+        $ret .= "<form method='get'>\n";
+        $ret .= LJ::make_authas_select($u, { authas => $GET{authas}, showall => 0 });
+        $ret .= "</form>\n\n";
+    }
+
+    $ret .= "<p class='intro'>" . BML::ml( '.intro', { sitename => $LJ::SITENAMESHORT, user => $remote->ljuser_display } ) . "</p>";
     $ret .= LJ::Widget::ImportQueueStatus->render;
 
     if ( LJ::Widget::ImportStatus->should_render ) {
diff -r e901769c10a1 -r 99ffce26d485 htdocs/tools/importer.bml.text
--- a/htdocs/tools/importer.bml.text	Sat Dec 24 16:23:51 2011 +0800
+++ b/htdocs/tools/importer.bml.text	Mon Dec 26 07:56:11 2011 +0000
@@ -1,4 +1,6 @@
 ;; -*- coding: utf-8 -*-
+.error.cant_import_comm=Community imports are not currently available for the chosen community.
+
 .error.notperson=You must be logged in to a personal account (not OpenID) in order to import content into it.
 
 .intro<<
--------------------------------------------------------------------------------