mark: A photo of Mark kneeling on top of the Taal Volcano in the Philippines. It was a long hike. (Default)
Mark Smith ([staff profile] mark) wrote in [site community profile] changelog2011-12-26 07:52 am
  • Add Memory
  • Share This Entry

[dw-free] allow maintainers to import communities from other sites

[commit: http://hg.dwscoalition.org/dw-free/rev/99ffce26d485]

http://bugs.dwscoalition.org/show_bug.cgi?id=2391

Adds the ability to import communities from a remote site to a local
community. This is a pretty straightforward system on top of the existing
importer and follows many of the same rules.

Some notes:

* There is a configuration option to turn this on, as well as a user cap
that needs to be present.

* You can disable importing comments for accounts with more than X comments,
if you want to help with load.

* This requires that you be a maintainer of the community on the local site
and the remote site. (We don't want people arbitrarily importing communities
they don't own.)

Patch by [staff profile] mark.

Files modified:
  • bin/checkconfig.pl
  • cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm
  • cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm
  • cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm
  • cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm
  • cgi-bin/LJ/Protocol.pm
  • cgi-bin/LJ/Talk.pm
  • cgi-bin/LJ/User.pm
  • cgi-bin/LJ/Widget/ImportChooseData.pm
  • cgi-bin/LJ/Widget/ImportChooseSource.pm
  • cgi-bin/LJ/Widget/ImportStatus.pm
  • cvs/multicvs.conf
  • doc/config-local.pl.txt
  • htdocs/stc/importer.css
  • htdocs/tools/importer.bml
  • htdocs/tools/importer.bml.text
--------------------------------------------------------------------------------
diff -r e901769c10a1 -r 99ffce26d485 bin/checkconfig.pl
--- a/bin/checkconfig.pl	Sat Dec 24 16:23:51 2011 +0800
+++ b/bin/checkconfig.pl	Mon Dec 26 07:56:11 2011 +0000
@@ -237,6 +237,7 @@
                     deb => 'libio-aoi-perl',
                     opt => 'Required for Perlbal',
                  },
+               "LWPx::ParanoidAgent" => { deb => 'liblwpx-paranoidagent-perl' },
               );
 
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Comments.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -221,6 +221,12 @@
             }
         );
         $parser->parse( $content );
+
+        # this is the best place to test for too many comments. if this site is limiting
+        # the comment imports for some reason or another, we can bail here.
+        return $fail->( $LJ::COMMENT_IMPORT_ERROR || 'Too many comments to import.' )
+            if defined $LJ::COMMENT_IMPORT_MAX && defined $server_max_id &&
+                $server_max_id > $LJ::COMMENT_IMPORT_MAX;
     }
     $log->( 'Finished fetching metadata.' );
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/FriendGroups.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -76,7 +76,7 @@
         q{UPDATE import_items SET status = 'ready'
           WHERE userid = ? AND item IN ('lj_friends', 'lj_entries')
           AND import_data_id = ? AND status = 'init'},
-        undef, $u->id, $opts->{import_data_id}        
+        undef, $u->id, $opts->{import_data_id}
     );
 
     return $ok->();
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Tags.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -53,6 +53,9 @@
         or return $fail->( 'Unable to load target with id %d.', $data->{userid} );
     $0 = sprintf( 'content-importer [tags: %s(%d)]', $u->user, $u->id );
 
+    my $dbh = LJ::get_db_writer()
+        or return $temp_fail->( 'Unable to get global master database handle' );
+
     # get tags
     my $r = $class->call_xmlrpc( $data, 'getusertags' );
     return $temp_fail->( 'XMLRPC failure: ' . $r->{faultString} )
@@ -60,6 +63,16 @@
 
     DW::Worker::ContentImporter::Local::Tags->merge_tags( $u, $r->{tags} );
 
+    # if this is a community, it is now our job to schedule the entry import
+    if ( $u->is_community ) {
+        $dbh->do(
+            q{UPDATE import_items SET status = 'ready'
+              WHERE userid = ? AND item = 'lj_entries'
+                  AND import_data_id = ? AND status = 'init'},
+            undef, $u->id, $opts->{import_data_id}
+        );
+    }
+
     return $ok->();
 }
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm
--- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Verify.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -77,6 +77,19 @@
     return $temp_fail->( 'XMLRPC failure: ' . $r->{faultString} )
         if ! $r || $r->{fault};
 
+    # If this is a community import, we have to do a second step now to make sure
+    # that the user is an administrator of the remote community. The best way I can
+    # come up with is try to unban the owner -- if it works, you're an admin.
+    if ( $data->{usejournal} ) {
+        $r = $class->call_xmlrpc( $data, 'consolecommand', {
+            commands => [ "ban_unset $data->{username} from $data->{usejournal}" ],
+        } );
+        return $temp_fail->( 'XMLRPC failure: ' . $r->{faultString} )
+            if ! $r || $r->{fault};
+        return $fail->( 'You are not an administrator/maintainer of the remote community.' )
+            unless $r->{results}->[0]->{output}->[0]->[0] eq 'success';
+    }
+
     # mark the next group as ready to schedule
     my $dbh = LJ::get_db_writer();
     $dbh->do(
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Protocol.pm
--- a/cgi-bin/LJ/Protocol.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Protocol.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -1361,6 +1361,14 @@
     unless ($req->{'props'}->{"opt_backdated"}) {
         $rlogtime -= $now;
     }
+    my $logtime = "FROM_UNIXTIME($now)";
+
+    # this is when the entry was posted. for most cases this is accurate but in case
+    # we're using the importer in the community case, it will mess life up.
+    if ( $importer_bypass && $posterid != $ownerid ) {
+        $logtime = $qeventtime;
+        $rlogtime = "$LJ::EndOfTime - UNIX_TIMESTAMP($qeventtime)";
+    }
 
     my $dupsig = Digest::MD5::md5_hex(join('', map { $req->{$_} }
                                            qw(subject event usejournal security allowmask)));
@@ -1523,7 +1531,7 @@
     my $dberr;
     $uowner->log2_do(\$dberr, "INSERT INTO log2 (journalid, jitemid, posterid, eventtime, logtime, security, ".
                      "allowmask, replycount, year, month, day, revttime, rlogtime, anum) ".
-                     "VALUES ($ownerid, $jitemid, $posterid, $qeventtime, FROM_UNIXTIME($now), $qsecurity, $qallowmask, ".
+                     "VALUES ($ownerid, $jitemid, $posterid, $qeventtime, $logtime, $qsecurity, $qallowmask, ".
                      "0, $req->{'year'}, $req->{'mon'}, $req->{'day'}, $LJ::EndOfTime-".
                      "UNIX_TIMESTAMP($qeventtime), $rlogtime, $anum)");
     return $fail->($err,501,$dberr) if $dberr;
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Talk.pm
--- a/cgi-bin/LJ/Talk.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Talk.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -1300,7 +1300,8 @@
                 }
                 $post->{picid} = $id;
                 $post->{pickw} = $kw;
-                push @load_pic, [ $pu, $id ];
+                push @load_pic, [ $pu, $id ]
+                    if defined $id;
             }
             load_userpics( $opts->{userpicref}, \@load_pic );
         }
@@ -1336,7 +1337,7 @@
     foreach my $row (@{$idlist})
     {
         my ($u, $id) = @$row;
-        next unless ref $u;
+        next unless ref $u && defined $id;
 
         if ($LJ::CACHE_USERPIC{$id}) {
             $upics->{$id} = $LJ::CACHE_USERPIC{$id};
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/User.pm
--- a/cgi-bin/LJ/User.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/User.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -2010,6 +2010,10 @@
     return $_[0]->get_cap( 'disable_can_post' ) ? 1 : 0;
 }
 
+sub can_import_comm {
+    return $_[0]->get_cap( 'import_comm' ) ? 1 : 0;
+}
+
 sub can_receive_vgifts_from {
     my ( $u, $remote, $is_anon ) = @_;
     $remote ||= LJ::get_remote();
@@ -3193,7 +3197,6 @@
 
     my ($url, $name);
     if ($id->typeid eq 'O') {
-        require Net::OpenID::Consumer;
         $url = $id->value;
         $name = Net::OpenID::VerifiedIdentity::DisplayOfURL($url, $LJ::IS_DEV_SERVER);
         $name = LJ::Hooks::run_hook("identity_display_name", $name) || $name;
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Widget/ImportChooseData.pm
--- a/cgi-bin/LJ/Widget/ImportChooseData.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Widget/ImportChooseData.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -34,12 +34,28 @@
             display_name => $class->ml( 'widget.importstatus.item.lj_bio' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_bio.desc' ),
             selected => 0,
+            comm_okay => 1,
+        },
+        {
+            name => 'lj_friends',
+            display_name => $class->ml( 'widget.importstatus.item.lj_friends' ),
+            desc => $class->ml( 'widget.importchoosedata.item.lj_friends.desc' ),
+            selected => 0,
+            comm_okay => 0,
+        },
+        {
+            name => 'lj_friendgroups',
+            display_name => $class->ml( 'widget.importstatus.item.lj_friendgroups' ),
+            desc => $class->ml( 'widget.importchoosedata.item.lj_friendgroups.desc', { sitename => $LJ::SITENAMESHORT } ),
+            selected => 0,
+            comm_okay => 0,
         },
         {
             name => 'lj_entries',
             display_name => $class->ml( 'widget.importstatus.item.lj_entries' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_entries.desc' ),
             selected => 0,
+            comm_okay => 1,
 
             suboptions => [
                 {
@@ -51,34 +67,25 @@
             ]
         },
         {
-            name => 'lj_friends',
-            display_name => $class->ml( 'widget.importstatus.item.lj_friends' ),
-            desc => $class->ml( 'widget.importchoosedata.item.lj_friends.desc' ),
-            selected => 0,
-        },
-        {
             name => 'lj_comments',
             display_name => $class->ml( 'widget.importstatus.item.lj_comments' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_comments.desc' ),
             selected => 0,
+            comm_okay => 1,
         },
         {
             name => 'lj_tags',
             display_name => $class->ml( 'widget.importstatus.item.lj_tags' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_tags.desc' ),
             selected => 0,
+            comm_okay => 1,
         },
         {
             name => 'lj_userpics',
             display_name => $class->ml( 'widget.importstatus.item.lj_userpics' ),
             desc => $class->ml( 'widget.importchoosedata.item.lj_userpics.desc', { sitename => $LJ::SITENAMESHORT } ),
             selected => 0,
-        },
-        {
-            name => 'lj_friendgroups',
-            display_name => $class->ml( 'widget.importstatus.item.lj_friendgroups' ),
-            desc => $class->ml( 'widget.importchoosedata.item.lj_friendgroups.desc', { sitename => $LJ::SITENAMESHORT } ),
-            selected => 0,
+            comm_okay => 1,
         },
     );
 
@@ -90,6 +97,8 @@
     $ret .= "<div class='importoptions'>";
 
     foreach my $option ( @options ) {
+        next unless $u->is_person || $option->{comm_okay};
+
         $ret .= "<div class='importoption'>";
         $ret .= $class->html_check(
             name => $option->{name},
@@ -166,6 +175,12 @@
     # everybody needs a verifier
     $post->{lj_verify} = 1;
 
+    # and finally, make sure modes that make no sense for communities are off
+    if ( $u->is_community ) {
+        $post->{lj_friends} = 0;
+        $post->{lj_friendgroups} = 0;
+    }
+
     return ( ret => LJ::Widget::ImportConfirm->render( %$post ) );
 }
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Widget/ImportChooseSource.pm
--- a/cgi-bin/LJ/Widget/ImportChooseSource.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Widget/ImportChooseSource.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -88,7 +88,7 @@
     $ret .= "<label for='password'>" . $class->ml( 'widget.importchoosesource.password' ) . "</label>";
     $ret .= $class->html_text( type => 'password', name => 'password' );
     $ret .= "</div>";
-    if ( $LJ::ALLOW_COMM_IMPORT{$u->user} ) {
+    if ( $u->is_community ) {
         $ret .= "<div class='i-usejournal'>";
         $ret .= "<label for='usejournal'>" . $class->ml( 'widget.importchoosesource.usejournal' ) . "</label>";
         $ret .= $class->html_text( name => 'usejournal', maxlength => 255 );
@@ -117,13 +117,13 @@
     $un =~ s/-/_/g;
 
     # be sure to sanitize the usejournal
-    my $uj = undef;
-    if ( $LJ::ALLOW_COMM_IMPORT{$u->user} ) {
+    my $uj;
+    if ( $u->is_community ) {
         $uj = LJ::trim( lc $post->{usejournal} );
         $uj =~ s/-/_/g;
     }
 
-    my $pw = $post->{password};
+    my $pw = LJ::trim( $post->{password} );
     return ( ret => $class->ml( 'widget.importchoosesource.error.nocredentials' ) )
         unless $un && $pw;
 
diff -r e901769c10a1 -r 99ffce26d485 cgi-bin/LJ/Widget/ImportStatus.pm
--- a/cgi-bin/LJ/Widget/ImportStatus.pm	Sat Dec 24 16:23:51 2011 +0800
+++ b/cgi-bin/LJ/Widget/ImportStatus.pm	Mon Dec 26 07:56:11 2011 +0000
@@ -42,7 +42,7 @@
             my $import_item = $items->{$importid};
 
             $ret .= "<tr><td colspan='4' class='table-header'>" . $class->ml( 'widget.importstatus.whichaccount', { user => $import_item->{user}, host => $import_item->{host} } ) . " | ";
-            $ret .= "<a href='$LJ::SITEROOT/tools/importer'>" . $class->ml( 'widget.importstatus.refresh' ) . "</a></td></tr>";
+            $ret .= "<a href='$LJ::SITEROOT/tools/importer?authas=" . $u->user . "'>" . $class->ml( 'widget.importstatus.refresh' ) . "</a></td></tr>";
             foreach my $item ( sort keys %{$import_item->{items}} ) {
                 my $i = $import_item->{items}->{$item};
                 my $color = { init => '#333333', ready => '#3333aa', queued => '#33aa33', failed => '#aa3333', succeeded => '#00ff00' }->{$i->{status}};
diff -r e901769c10a1 -r 99ffce26d485 cvs/multicvs.conf
--- a/cvs/multicvs.conf	Sat Dec 24 16:23:51 2011 +0800
+++ b/cvs/multicvs.conf	Mon Dec 26 07:56:11 2011 +0000
@@ -23,7 +23,6 @@
 SVN(memcached)            = http://code.livejournal.org/svn/memcached/trunk/
 SVN(CSS-Cleaner)          = http://code.livejournal.org/svn/CSS-Cleaner/trunk/
 SVN(Sys-Syscall)          = http://code.livejournal.org/svn/Sys-Syscall/trunk/
-SVN(LWPx-ParanoidAgent)   = http://code.livejournal.org/svn/LWPx-ParanoidAgent/trunk/
 SVN(Danga-Socket)         = http://code.livejournal.org/svn/Danga-Socket/trunk/
 SVN(mogilefs)             = http://code.sixapart.com/svn/mogilefs/trunk/ @1459
 SVN(TheSchwartz)          = http://code.sixapart.com/svn/TheSchwartz/trunk/
@@ -46,7 +45,6 @@
 TheSchwartz-Worker-SendEmail/lib              cgi-bin/
 Sys-Syscall/lib                               cgi-bin
 Danga-Socket/lib/Danga/Socket.pm              cgi-bin/Danga/Socket.pm
-LWPx-ParanoidAgent/lib/LWPx                   cgi-bin/LWPx
 #openid/perl/Net-OpenID-Consumer/lib           cgi-bin
 #openid/perl/Net-OpenID-Server/lib             cgi-bin
 #openid/perl/Net-OpenID-Common/lib             cgi-bin
diff -r e901769c10a1 -r 99ffce26d485 doc/config-local.pl.txt
--- a/doc/config-local.pl.txt	Sat Dec 24 16:23:51 2011 +0800
+++ b/doc/config-local.pl.txt	Mon Dec 26 07:56:11 2011 +0000
@@ -113,6 +113,14 @@
     #    rename => [ 15, undef, undef, 150 ],
     #);
 
+    # You can turn on/off community importing here.
+    $ALLOW_COMM_IMPORTS = 0;
+
+    # If this is defined and a number, if someone tries to import more than this many
+    # comments in a single import, the error specified will be raised and the job will fail.
+    $COMMENT_IMPORT_MAX = undef;
+    $COMMENT_IMPORT_ERROR = "Importing more than 10,000 comments is currently disabled.";
+
 }
 
 1;
diff -r e901769c10a1 -r 99ffce26d485 htdocs/stc/importer.css
--- a/htdocs/stc/importer.css	Sat Dec 24 16:23:51 2011 +0800
+++ b/htdocs/stc/importer.css	Mon Dec 26 07:56:11 2011 +0000
@@ -68,6 +68,32 @@
     margin-left: -10px;
 }
 
+.importer .usejournal {
+    border-top: 1px solid #CCC;
+    padding: 10px;
+    margin-bottom: 10px;
+}
+
+.importer .usejournal select {
+    position: absolute;
+    left: 22em;
+    margin-top: -1px;
+}
+
+.importer .usejournal input {
+    position: absolute;
+    left: 22em;
+    margin-top: -1px;
+}
+
+.importer .usejournal label {
+    font-weight: bold;
+}
+
+.importer .usejournal div {
+    margin-bottom: 5px;
+}
+
 .importer .credentials {
     border-top: 1px solid #CCC;
     padding: 10px;
@@ -97,7 +123,7 @@
 }
 
 .importer .importantnote {
-    color: #666;
+    color: #667;
     font-size: 0.9em;
     padding: 10px;
     border-top: 1px solid #CCC;
diff -r e901769c10a1 -r 99ffce26d485 htdocs/tools/importer.bml
--- a/htdocs/tools/importer.bml	Sat Dec 24 16:23:51 2011 +0800
+++ b/htdocs/tools/importer.bml	Mon Dec 26 07:56:11 2011 +0000
@@ -32,11 +32,19 @@
 
     $title = $ML{'.title'};
 
-    my $remote = LJ::get_remote()
-        or return "<?needlogin?>";
+    my $u = LJ::get_remote();
+    my $remote = LJ::get_effective_remote();
+    return "<?needlogin?>" unless $u && $remote;
+
+    # if these aren't the same users, make sure we're allowed to do a community import
+    unless ( $u->equals( $remote ) ) {
+        return $ML{'.error.cant_import_comm'}
+            unless $LJ::ALLOW_COMM_IMPORTS &&
+                ( $u->can_import_comm || $remote->can_import_comm );
+    }
 
     return $ML{'.error.notperson'}
-        unless $remote->is_person;
+        unless $u->is_person;
 
     if ( LJ::did_post() ) {
         return "<?h1 $ML{'Error'} h1?><?p $ML{'error.invalidform'} p?>"
@@ -46,13 +54,22 @@
         if ( $from_post{notloggedin} ) {
             return "<?needlogin?>";
         } elsif ( $from_post{refresh} ) {
-            return BML::redirect( "$LJ::SITEROOT/tools/importer" );
+            my $authas = $remote->equals( $u ) ? "" : "?authas=" . $remote->user;
+            return BML::redirect( "$LJ::SITEROOT/tools/importer$authas" );
         } elsif ( $from_post{ret} ) {
             return "<div class='importer'>$from_post{ret}</div>";
         }
     }
 
-    my $ret = "<p class='intro'>" . BML::ml( '.intro', { sitename => $LJ::SITENAMESHORT, user => $remote->ljuser_display } ) . "</p>";
+    # user switcher
+    my $ret = '';
+    if ( $LJ::ALLOW_COMM_IMPORTS && $u->can_import_comm ) {
+        $ret .= "<form method='get'>\n";
+        $ret .= LJ::make_authas_select($u, { authas => $GET{authas}, showall => 0 });
+        $ret .= "</form>\n\n";
+    }
+
+    $ret .= "<p class='intro'>" . BML::ml( '.intro', { sitename => $LJ::SITENAMESHORT, user => $remote->ljuser_display } ) . "</p>";
     $ret .= LJ::Widget::ImportQueueStatus->render;
 
     if ( LJ::Widget::ImportStatus->should_render ) {
diff -r e901769c10a1 -r 99ffce26d485 htdocs/tools/importer.bml.text
--- a/htdocs/tools/importer.bml.text	Sat Dec 24 16:23:51 2011 +0800
+++ b/htdocs/tools/importer.bml.text	Mon Dec 26 07:56:11 2011 +0000
@@ -1,4 +1,6 @@
 ;; -*- coding: utf-8 -*-
+.error.cant_import_comm=Community imports are not currently available for the chosen community.
+
 .error.notperson=You must be logged in to a personal account (not OpenID) in order to import content into it.
 
 .intro<<
--------------------------------------------------------------------------------
  • Add Memory
  • Share This Entry
(17 comments)

Post a comment in response:

This account has disabled anonymous posting.
If you don't have an account you can create one now.
HTML doesn't work in the subject.
More info about formatting

If you are unable to use this captcha for any reason, please contact us by email at support@dreamwidth.org

 
Notice: This account is set to log the IP addresses of everyone who comments.
Links will be displayed as unclickable URLs to help prevent spam.