[dw-free] New dversion for userpics to make renaming userpics faster
[commit: http://hg.dwscoalition.org/dw-free/rev/f897918203ac]
http://bugs.dwscoalition.org/show_bug.cgi?id=3064
Update userpic schema and add a migration tool to convert users from the old
to the new. We'll be rolling this out slowly and with a lot of checks to
make sure we get it right.
Patch by
exor674.
Files modified:
http://bugs.dwscoalition.org/show_bug.cgi?id=3064
Update userpic schema and add a migration tool to convert users from the old
to the new. We'll be rolling this out slowly and with a lot of checks to
make sure we get it right.
Patch by
![[personal profile]](https://www.dreamwidth.org/img/silk/identity/user.png)
Files modified:
- bin/upgrading/d8d9-userpicrename.pl
- bin/upgrading/proplists.dat
- bin/upgrading/update-db-general.pl
- cgi-bin/DW/External/XPostProtocol/LJXMLRPC.pm
- cgi-bin/DW/User/DVersion/Migrate8To9.pm
- cgi-bin/DW/Worker/ContentImporter/LiveJournal/Entries.pm
- cgi-bin/DW/Worker/ContentImporter/Local/Comments.pm
- cgi-bin/LJ/Comment.pm
- cgi-bin/LJ/Constants.pm
- cgi-bin/LJ/Entry.pm
- cgi-bin/LJ/S2.pm
- cgi-bin/LJ/S2/EntryPage.pm
- cgi-bin/LJ/S2/ReplyPage.pm
- cgi-bin/LJ/Talk.pm
- cgi-bin/LJ/User.pm
- cgi-bin/LJ/Userpic.pm
- cgi-bin/ljlib.pl
- cgi-bin/ljprotocol.pl
- htdocs/admin/entryprops.bml
- htdocs/talkpost.bml
- htdocs/talkread.bml
-------------------------------------------------------------------------------- diff -r ebea5b13e8a4 -r f897918203ac bin/upgrading/d8d9-userpicrename.pl --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/bin/upgrading/d8d9-userpicrename.pl Sat Oct 02 01:27:29 2010 +0800 @@ -0,0 +1,148 @@ +#!/usr/bin/perl +# +# This code was forked from the LiveJournal project owned and operated +# by Live Journal, Inc. The code has been modified and expanded by +# Dreamwidth Studios, LLC. These files were originally licensed under +# the terms of the license supplied by Live Journal, Inc, which can +# currently be found at: +# +# http://code.livejournal.org/trac/livejournal/browser/trunk/LICENSE-LiveJournal.txt +# +# In accordance with the original license, this code and all its +# modifications are provided under the GNU General Public License. +# A copy of that license can be found in the LICENSE file included as +# part of this distribution. +# +# Goes over every user, updating their dversion to 9 and +# moves userpicmap2 over to userpicmap3 +# +use strict; +use warnings; +BEGIN { + use lib "$ENV{'LJHOME'}/cgi-bin/"; + require "ljlib.pl"; +} +use Term::ReadLine; +use Getopt::Long; +use DW::User::DVersion::Migrate8To9; + +my $BLOCK_SIZE = 10_000; # get users in blocks of 10,000 +my $VERBOSE = 0; # print out extra info +my $need_help; +my @cluster; +my @users; +my $endtime; + +my $help = <<"END"; +Usage: $0 [options] +Options: + --cluster=N Specify user cluster to work on (by default, all clusters) + --hours=N Work no more than N hours (by default, work until all is done) + --user=N Specify users to migrate (by default, all users on the specified clusters) + --verbose Be noisy + --help Print this help and exit +END + +GetOptions( + "help" => \$need_help, + "cluster=i" => \@cluster, + "user=s" => \@users, + "verbose" => \$VERBOSE, + "hours=i" => sub { $endtime = $_[1]*3600+time(); }, +) or die $help; + +if ( $need_help ) { + print $help; + exit(0); +} + +unless ( @cluster ) { + no warnings 'once'; + @cluster = ( 0, @LJ::CLUSTERS ); +} + +my $dbh = LJ::get_db_writer() + or die "Could not connect to global master"; + +my $users = join( ', ', map { $dbh->quote($_) } @users ); + +my $term = new Term::ReadLine 'd8-d9 migrator'; +my $line = $term->readline( "Do you want to update to dversion 9 (userpicmap3)? [N/y] " ); +unless ( $line =~ /^y/i ) { + print "Not upgrading to dversion 9\n\n"; + exit; +} + +print "\n--- Upgrading users to dversion (userpicmap3) ---\n\n"; + +# get user count +my $total = $dbh->selectrow_array( "SELECT COUNT(*) FROM user WHERE dversion = 8" ); +print "\tTotal users at dversion 8: $total\n\n"; + +my $migrated = 0; +my $flag_stop_work = 0; + +MAIN_LOOP: +foreach my $cid ( @cluster ) { + + while ( 1 ) { + my $sth; + if ( @users ) { + $sth = $dbh->prepare( "SELECT userid FROM user WHERE dversion=8 AND clusterid=? AND user IN ($users) LIMIT $BLOCK_SIZE" ); + } else { + $sth = $dbh->prepare( "SELECT userid FROM user WHERE dversion=8 AND clusterid=? LIMIT $BLOCK_SIZE" ); + } + $sth->execute( $cid ); + die $sth->errstr if $sth->err; + + my $count = $sth->rows; + print "\tGot $count users on cluster $cid with dversion=8\n"; + last unless $count; + + local( $SIG{TERM}, $SIG{INT}, $SIG{HUP} ); + $SIG{TERM} = $SIG{INT} = $SIG{HUP} = sub { $flag_stop_work = 1; }; + while ( my ( $userid ) = $sth->fetchrow_array ) { + if ( $flag_stop_work ) { + warn "Exiting by signal..."; + last MAIN_LOOP; + } + if ( $endtime && time()>$endtime ) { + warn "Exiting by time condition..."; + last MAIN_LOOP; + } + + my $u = LJ::load_userid( $userid ) + or die "Invalid userid: $userid"; + + if ( $u->is_expunged ) { + ## special case: expunged (deleted) users + ## just update dbversion, don't move or delete(?) data + $u->update_self( { 'dversion' => 9 } ); + print "\tUpgrading version of deleted user $u->{user}\n" if $VERBOSE; + $migrated++; + } else { + # lock while upgrading + my $lock = LJ::locker()->trylock( "d8d9-$userid" ); + unless ( $lock ) { + print STDERR "Could not get a lock for user " . $u->user . ".\n"; + next; + } + + my $ok = eval { $u->upgrade_to_dversion_9 }; + die $@ if $@; + + print "\tMigrated user " . $u->user . "... " . ($ok ? 'ok' : 'ERROR') . "\n" + if $VERBOSE; + + $migrated++ if $ok; + } + } + + print "\t - Migrated $migrated users so far\n\n"; + + # make sure we don't end up running forever for whatever reason + last if $migrated > $total; + } +} + +print "--- Done migrating $migrated of $total users to dversion 9 ---\n"; diff -r ebea5b13e8a4 -r f897918203ac bin/upgrading/proplists.dat --- a/bin/upgrading/proplists.dat Sat Oct 02 00:29:19 2010 +0800 +++ b/bin/upgrading/proplists.dat Sat Oct 02 01:27:29 2010 +0800 @@ -1161,6 +1161,11 @@ talkproplist.picture_keyword: des: A keyword that should align to a defined picture prettyname: Picture Keyword +talkproplist.picture_mapid: + datatype: num + des: A keyword that should align to a mapid + prettyname: Picture Keyword MapID + talkproplist.poster_ip: datatype: char des: The poster's IP address, optionally logged. @@ -1324,6 +1329,13 @@ logproplist.picture_keyword: sortorder: 30 ownership: user +logproplist.picture_mapid: + datatype: num + des: A keyword that should align to a mapid + prettyname: Picture Keyword MapID + sortorder: 30 + ownership: system + logproplist.revnum: datatype: num des: Number of times this post has been edited. diff -r ebea5b13e8a4 -r f897918203ac bin/upgrading/update-db-general.pl --- a/bin/upgrading/update-db-general.pl Sat Oct 02 00:29:19 2010 +0800 +++ b/bin/upgrading/update-db-general.pl Sat Oct 02 01:27:29 2010 +0800 @@ -554,6 +554,20 @@ CREATE TABLE userpicmap2 ( picid int(10) unsigned NOT NULL default '0', PRIMARY KEY (userid, kwid) +) +EOC + +register_tablecreate("userpicmap3", <<'EOC'); +CREATE TABLE userpicmap3 ( + userid int(10) unsigned NOT NULL default '0', + mapid int(10) unsigned NOT NULL, + kwid int(10) unsigned, + picid int(10) unsigned, + redirect_mapid int(10) unsigned, + + PRIMARY KEY (userid, mapid), + UNIQUE KEY (userid, kwid), + INDEX redirect (userid, redirect_mapid) ) EOC diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/DW/External/XPostProtocol/LJXMLRPC.pm --- a/cgi-bin/DW/External/XPostProtocol/LJXMLRPC.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/DW/External/XPostProtocol/LJXMLRPC.pm Sat Oct 02 01:27:29 2010 +0800 @@ -315,9 +315,12 @@ sub entry_to_req { my $entryprops = $entry->props; $req->{props} = {}; # only bring over these properties - for my $entrykey (qw ( adult_content current_coords current_location current_music opt_backdated opt_nocomments opt_noemail opt_preformatted opt_screening picture_keyword taglist used_rte pingback )) { + for my $entrykey (qw ( adult_content current_coords current_location current_music opt_backdated opt_nocomments opt_noemail opt_preformatted opt_screening taglist used_rte pingback )) { $req->{props}->{$entrykey} = $entryprops->{$entrykey} if defined $entryprops->{$entrykey}; } + + # and regenerate this one from data + $req->{props}->{picture_keyword} = $entry->userpic_kw; # figure out what current_moodid and current_mood to pass to the crossposted entry my ( $siteid, $moodid, $mood ) = ( $extacct->siteid, $entryprops->{current_moodid}, $entryprops->{current_mood} ); diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/DW/User/DVersion/Migrate8To9.pm --- /dev/null Thu Jan 01 00:00:00 1970 +0000 +++ b/cgi-bin/DW/User/DVersion/Migrate8To9.pm Sat Oct 02 01:27:29 2010 +0800 @@ -0,0 +1,265 @@ +#!/usr/bin/perl +# +# DW::User::DVersion::Migrate8To9 - Handling dversion 8 to 9 migration +# +# Authors: +# Andrea Nall <anall@andreanall.com> +# +# Copyright (c) 2010 by Dreamwidth Studios, LLC. +# +# This program is free software; you may redistribute it and/or modify it under +# the same terms as Perl itself. For a copy of the license, please reference +# 'perldoc perlartistic' or 'perldoc perlgpl'. + +package DW::User::DVersion::Migrate8To9; + +use strict; +use warnings; +use LJ::User; +use Time::HiRes qw( usleep ); + +my $readonly_bit; + +# find readonly cap class, complain if not found +foreach (keys %LJ::CAP) { + if ($LJ::CAP{$_}->{'_name'} eq "_moveinprogress" && + $LJ::CAP{$_}->{'readonly'} == 1) { + $readonly_bit = $_; + last; + } +} +unless (defined $readonly_bit) { + die "Won't move user without %LJ::CAP capability class named '_moveinprogress' with readonly => 1\n"; +} + +my $logpropid = LJ::get_prop( log => 'picture_keyword' )->{id}; +my $talkpropid = LJ::get_prop( talk => 'picture_keyword' )->{id}; + +my $logpropid_map = LJ::get_prop( log => 'picture_mapid' )->{id}; +my $talkpropid_map = LJ::get_prop( talk => 'picture_mapid' )->{id}; + +sub do_upgrade { + my $BLOCK_INSERT = 25; + + my ( $u ) = @_; + + return 0 if $u->readonly; + + return 1 if $u->dversion >= 9; + + # we really cannot have the user doing things during this process + $u->modify_caps( [$readonly_bit], [] ); + + # wait a quarter of a second, give any request the user might be doing a chance to stop + # as the user changing things could lead to slight data loss re. userpic selection + # on entries and comments + usleep( 250000 ); + + # do this in an eval so, in case something dies, we don't leave the user locked + my $rv = 0; + + eval { + # Unfortunately, we need to iterate over all clusters to get a list + # of used keywords so we can give proper ids to everything, + # even removed keywords + my %keywords; + my %to_update; + if ( $u->is_individual ) { + foreach my $cluster_id ( @LJ::CLUSTERS ) { + my $dbcm_o = LJ::get_cluster_master( $cluster_id ); + + my $entries = $dbcm_o->selectall_arrayref(q{ + SELECT log2.journalid AS journalid, + log2.jitemid AS jitemid, + logprop2.value AS value + FROM logprop2 + INNER JOIN log2 + ON ( logprop2.journalid = log2.journalid + AND logprop2.jitemid = log2.jitemid ) + WHERE posterid = ? + AND propid=? + }, undef, $u->id, $logpropid); + die $dbcm_o->errstr if $dbcm_o->err; + my $comments = $dbcm_o->selectall_arrayref( q{ + SELECT talkprop2.journalid AS journalid, + talkprop2.jtalkid AS jtalkid, + talkprop2.value AS value + FROM talkprop2 + INNER JOIN talk2 + ON ( talkprop2.journalid = talk2.journalid + AND talkprop2.jtalkid = talk2.jtalkid ) + WHERE posterid = ? + AND tpropid=? + }, undef, $u->id, $talkpropid); + die $dbcm_o->errstr if $dbcm_o->err; + + $to_update{$cluster_id} = { + entries => $entries, + comments => $comments, + }; + $keywords{$_->[2]}->{count}++ foreach ( @$entries, @$comments ); + } + } + + my $origmap = $u->selectall_hashref( q{ + SELECT kwid, picid FROM userpicmap2 WHERE userid=? + }, "kwid", undef, $u->id); + die $u->errstr if $u->err; + + my $picmap = $u->selectall_hashref( q{ + SELECT picid, state FROM userpic2 WHERE userid=? + }, "picid", undef, $u->id); + die $u->errstr if $u->err; + + my %outrows; + + my %kwid_map; + + foreach my $k ( keys %keywords ) { + if ( $k =~ m/^pic#(\d+)$/ ) { + my $picid = $1; + next if ! exists $picmap->{$picid} || $picmap->{$picid}->{state} eq 'X'; + $keywords{$k}->{kwid} = undef; + $keywords{$k}->{picid} = $picid; + $outrows{$picid}->{0}++; + } else { + my $kwid = $u->get_keyword_id($k,1); + $kwid_map{$kwid} = $k; + my $picid = $origmap->{$kwid}->{picid}; + $keywords{$k}->{kwid} = $kwid; + $keywords{$k}->{picid} = $picid; + $outrows{$picid || 0}->{$kwid}++; + } + } + + foreach my $r ( values %$origmap ) { + $outrows{$r->{picid}}->{$r->{kwid}}++ if $r->{picid} && $r->{kwid}; + } + + { + my ( @bind, @vals ); + + # flush rows to destination table + my $flush = sub { + return unless @bind; + + # insert data + my $bind = join( ",", @bind ); + $u->do( "REPLACE INTO userpicmap3 (userid,mapid,kwid,picid) VALUES $bind", undef, @vals ); + die $u->errstr if $u->err; + + # reset values + @bind = (); + @vals = (); + }; + + foreach my $picid ( sort { $a <=> $b } keys %outrows ) { + foreach my $kwid ( sort { $a <=> $b } keys %{$outrows{$picid}} ) { + next if $kwid == 0 && $picid == 0; + push @bind, "(?,?,?,?)"; + my $mapid = LJ::alloc_user_counter( $u, 'Y' ); + my $keyword = $kwid == 0 ? "pic#$picid" : $kwid_map{$kwid}; + # if $keyword is undef, this isn't used on any entries, so we don't care about the mapid + # however, if $kwid is undef, this is a pic#xxx keyword, and had to have existed on an entry + $keywords{$keyword}->{mapid} = $mapid if defined $keyword; + push @vals, ( $u->id, $mapid, $kwid || undef, $picid || undef ); + $flush->() if @bind > $BLOCK_INSERT; + } + } + $flush->(); + } + + if ( $u->is_individual ) { + foreach my $cluster_id ( @LJ::CLUSTERS ) { + next unless $to_update{$cluster_id}; + my $data = $to_update{$cluster_id}; + + my $dbcm_o = LJ::get_cluster_master($cluster_id); + + { + my ( @bind, @vals ); + + # flush rows to destination table + my $flush = sub { + return unless @bind; + + # insert data + my $bind = join( ",", @bind ); + $dbcm_o->do( "REPLACE INTO logprop2 (journalid,jitemid,propid,value) VALUES $bind", undef, @vals ); + die $u->errstr if $u->err; + + # reset values + @bind = (); + @vals = (); + }; + + foreach my $entry ( @{ $data->{entries} } ) { + next unless $keywords{$entry->[2]}->{mapid}; + push @bind, "(?,?,?,?)"; + push @vals, ( $entry->[0], $entry->[1], $logpropid_map, $keywords{$entry->[2]}->{mapid} ); + $flush->() if @bind > $BLOCK_INSERT; + } + $flush->(); + + foreach my $entry ( @{ $data->{entries} } ) { + LJ::MemCache::delete([$entry->[0],"logprop:". $entry->[0] .":". $entry->[1]]); + } + } + { + my ( @bind, @vals ); + + # flush rows to destination table + my $flush = sub { + return unless @bind; + + # insert data + my $bind = join( ",", @bind ); + $dbcm_o->do( "REPLACE INTO talkprop2 (journalid,jtalkid,tpropid,value) VALUES $bind", undef, @vals ); + die $u->errstr if $u->err; + + # reset values + @bind = (); + @vals = (); + }; + + foreach my $comment ( @{ $data->{comments} } ) { + next unless $keywords{$comment->[2]}->{mapid}; + push @bind, "(?,?,?,?)"; + push @vals, ( $comment->[0], $comment->[1], $talkpropid_map, $keywords{$comment->[2]}->{mapid} ); + $flush->() if @bind > $BLOCK_INSERT; + } + $flush->(); + + foreach my $comment ( @{ $data->{comments} } ) { + LJ::MemCache::delete([$comment->[0],"talkprop:". $comment->[0] .":". $comment->[1]]); + } + } + } + } + + $rv = 1; + }; + + my $err = $@; + + # okay, we're done, the user can do things again + $u->modify_caps( [], [$readonly_bit] ); + + die $err if $err; + + return $rv; +} + +sub upgrade_to_dversion_9 { + # If user has been purged, go ahead and update version + # Otherwise move their data + my $ok = $_[0]->is_expunged ? 1 : do_upgrade(@_); + + $_[0]->update_self( { 'dversion' => 9 } ) if $ok; + + LJ::Userpic->delete_cache( $_[0] ); + + return $ok; +} + +*LJ::User::upgrade_to_dversion_9 = \&upgrade_to_dversion_9; diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/DW/Worker/ContentImporter/LiveJournal/Entries.pm --- a/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Entries.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/DW/Worker/ContentImporter/LiveJournal/Entries.pm Sat Oct 02 01:27:29 2010 +0800 @@ -207,7 +207,12 @@ sub try_work { # local picture keyword if ( my $jitemid = $entry_map->{$evt->{key}} ) { my $entry = LJ::Entry->new( $u, jitemid => $jitemid ); - $entry->set_prop( picture_keyword => $evt->{props}->{picture_keyword} ); + my $kw = $evt->{props}->{picture_keyword}; + if ( $u->userpic_have_mapid ) { + $entry->set_prop( picture_mapid => $u->get_mapid_from_keyword( $kw, create => 1) ); + } else { + $entry->set_prop( picture_keyword => $kw ); + } } # now try to skip it diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/DW/Worker/ContentImporter/Local/Comments.pm --- a/cgi-bin/DW/Worker/ContentImporter/Local/Comments.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/DW/Worker/ContentImporter/Local/Comments.pm Sat Oct 02 01:27:29 2010 +0800 @@ -71,7 +71,12 @@ sub update_comment { # edits and such. for now, I'm just trying to get the icons to update... my $c = LJ::Comment->instance( $u, jtalkid => $cmt->{id} ) or return $$errref = 'Unable to instantiate LJ::Comment object.'; - $c->set_prop( picture_keyword => $cmt->{props}->{picture_keyword} ); + my $pu = $c->poster; + if ( $pu && $pu->userpic_have_mapid ) { + $c->set_prop( picture_mapid => $u->get_mapid_from_keyword( $cmt->{props}->{picture_keyword}, create => 1 ) ); + } else { + $c->set_prop( picture_keyword => $cmt->{props}->{picture_keyword} ); + } } diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/Comment.pm --- a/cgi-bin/LJ/Comment.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/Comment.pm Sat Oct 02 01:27:29 2010 +0800 @@ -26,6 +26,14 @@ use lib "$LJ::HOME/cgi-bin"; require "htmlcontrols.pl"; use LJ::Talk; + +=head1 NAME + +LJ::Comment + +=head1 CLASS METHODS + +=cut # internal fields: # @@ -218,6 +226,9 @@ sub create { } +=head1 INSTANCE METHODS + +=cut sub absorb_row { my ($self, %row) = @_; @@ -303,24 +314,6 @@ sub edit_url { my $url = $entry->url; return "$url?edit=$dtalkid"; -} - -# return img tag of userpic that the comment poster used -sub poster_userpic { - my $self = $_[0]; - my $pic_kw = $self->prop('picture_keyword'); - my $posteru = $self->poster; - - # anonymous poster, no userpic - return "" unless $posteru; - - # new from keyword falls back to the default userpic if - # there was no keyword, or if the keyword is no longer used - my $pic = LJ::Userpic->new_from_keyword($posteru, $pic_kw); - return $pic->imgtag_nosize if $pic; - - # no userpic with comment - return ""; } # return LJ::User of journal comment is in @@ -1383,10 +1376,8 @@ sub _format_mail_both { if ($is_html) { my $pichtml; - my $pic_kw = $self->prop('picture_keyword'); - if ( $posteru ) { - my $pic = LJ::Userpic->new_from_keyword( $posteru, $pic_kw ) || $posteru->userpic; + my ( $pic, $pic_kw ) = $self->userpic; if ( $pic && $pic->load_row ) { $pichtml = "<img src=\"$LJ::USERPIC_ROOT/$pic->{picid}/$pic->{userid}\" align='absmiddle' ". @@ -1553,20 +1544,45 @@ sub is_text_spam($\$) { return 0; # normal text } -# returns a LJ::Userpic object for the poster of the comment, or undef -# it will unify interface between Entry and Comment: $foo->userpic will -# work correctly for both Entry and Comment objects +=head2 C<< $cmt->userpic >> + +Returns a LJ::Userpic object for the poster of the comment, or undef. + +If called in a list context, returns ( LJ::Userpic object, keyword ) + +=cut sub userpic { my $self = $_[0]; my $up = $self->poster; return unless $up; - my $key = $self->prop('picture_keyword'); - # return the picture from keyword, if defined # else return poster's default userpic - return LJ::Userpic->new_from_keyword( $up, $key ) || $up->userpic; + my $kw = $_[0]->userpic_kw; + my $pic = LJ::Userpic->new_from_keyword( $up, $kw ) || $up->userpic; + + return wantarray ? ( $pic, $kw ) : $pic; +} + +=head2 C<< $cmt->userpic_kw >> + +Returns the userpic keyword used on this comment, or undef. + +=cut +sub userpic_kw { + my $self = $_[0]; + + my $up = $self->poster; + return unless $up; + + if ( $up->userpic_have_mapid ) { + my $mapid = $self->prop('picture_mapid'); + + return $up->get_keyword_from_mapid( $mapid ) if $mapid; + } else { + return $self->prop('picture_keyword'); + } } sub poster_ip { diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/Constants.pm --- a/cgi-bin/LJ/Constants.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/Constants.pm Sat Oct 02 01:27:29 2010 +0800 @@ -68,11 +68,12 @@ use constant CMAX_UPIC_DESCRIPTION => 12 # 6: clustered memories, friend groups, and keywords (for memories) # 7: clustered userpics, keyword limiting, and comment support # 8: clustered polls +# 9: userpicmap3, with mapid # # Dreamwidth installations should ALL be dversion >= 8. We do not support anything # else and are ripping out code to support all previous dversions. # -use constant MAX_DVERSION => 8; +use constant MAX_DVERSION => 9; $LJ::MAX_DVERSION = MAX_DVERSION; 1; diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/Entry.pm --- a/cgi-bin/LJ/Entry.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/Entry.pm Sat Oct 02 01:27:29 2010 +0800 @@ -21,6 +21,14 @@ use strict; use strict; use vars qw/ $AUTOLOAD /; use Carp qw/ croak confess /; + +=head1 NAME + +LJ::Entry + +=head1 CLASS METHODS + +=cut # internal fields: # @@ -186,6 +194,10 @@ sub new_from_row { return $self; } + +=head1 INSTANCE METHODS + +=cut # returns true if entry currently exists. (it's possible for a given # $u, to make a fake jitemid and that'd be a valid skeleton LJ::Entry @@ -854,35 +866,54 @@ sub tag_map { return $tags->{$self->jitemid} || {}; } -# returns a LJ::Userpic object for this post, or undef -# currently this is for the permalink view, not for the friends view -# context. TODO: add a context option for friends page, and perhaps +=head2 C<< $entry->userpic >> + +Returns a LJ::Userpic object for this post, or undef. + +If called in a list context, returns ( LJ::Userpic object, keyword ) + +See userpic_kw. + +=cut +# FIXME: add a context option for friends page, and perhaps # respect $remote's userpic viewing preferences (community shows poster # vs community's picture) sub userpic { - my $self = shift; + my $up = $_[0]->poster; + my $kw = $_[0]->userpic_kw; + my $pic = LJ::Userpic->new_from_keyword( $up, $kw ) || $up->userpic; + + return wantarray ? ( $pic, $kw ) : $pic; +} + +=head2 C<< $entry->userpic_kw >> + +Returns the keyword to use for the entry. + +If a keyword is specified, it uses that, otherwise +it tries the custom mood text, followed by the standard mood. + +=cut +sub userpic_kw { + my $self = $_[0]; my $up = $self->poster; + + my $key; + # try their entry-defined userpic keyword + if ( $up->userpic_have_mapid ) { + my $mapid = $self->prop('picture_mapid'); - # try their entry-defined userpic keyword, then their custom + $key = $up->get_keyword_from_mapid( $mapid ) if $mapid; + } else { + $key = $self->prop('picture_keyword'); + } + + # ... but if that fails, then their custom # mood, then their standard mood - my $key = $self->prop('picture_keyword') || - $self->prop('current_mood') || + return $key || $self->prop('current_mood') || DW::Mood->mood_name( $self->prop('current_moodid') ); - - # return the picture from keyword, if defined - # else return poster's default userpic - return LJ::Userpic->new_from_keyword( $up, $key ) || $up->userpic; } - -sub userpic_kw_from_props { - my ($class, $props) = @_; - - return $props->{'picture_keyword'} || - $props->{'current_mood'} || - DW::Mood->mood_name( $props->{'current_moodid'} ); -} - # returns true if the user is allowed to share an entry via Tell a Friend # $u is the logged-in user diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/S2.pm --- a/cgi-bin/LJ/S2.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/S2.pm Sat Oct 02 01:27:29 2010 +0800 @@ -1960,11 +1960,11 @@ sub Entry_from_entryobj # loading S2 Userpic my $userpic; - my $kw = $entry_obj->userpic_kw_from_props( $entry_obj->props ); + my ( $pic, $kw ) = $entry_obj->userpic; # if the post was made in a community, use either the userpic it was posted with or the community pic depending on the style setting if ( $posterid == $journalid || !S2::get_property_value($opts->{ctx}, 'use_shared_pic') ) { - $userpic = Image_userpic( $poster, $entry_obj->userpic->picid, $kw ) if $entry_obj->userpic; + $userpic = Image_userpic( $poster, $pic->picid, $kw ) if $pic; } else { $userpic = Image_userpic( $journal, $journal->userpic->picid ) if $journal->userpic; } diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/S2/EntryPage.pm --- a/cgi-bin/LJ/S2/EntryPage.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/S2/EntryPage.pm Sat Oct 02 01:27:29 2010 +0800 @@ -181,7 +181,7 @@ sub EntryPage $height = $height / 2; } - $comment_userpic = Image_userpic( $com->{upost}, $com->{picid}, $com->{props}->{picture_keyword}, + $comment_userpic = Image_userpic( $com->{upost}, $com->{picid}, $com->{pickw}, $width, $height ); } @@ -226,7 +226,7 @@ sub EntryPage '_type' => 'Comment', 'journal' => $userlite_journal, 'metadata' => { - 'picture_keyword' => $com->{'props'}->{'picture_keyword'}, + 'picture_keyword' => $com->{pickw}, }, 'permalink_url' => "$permalink?thread=$dtalkid" . LJ::Talk::comment_anchor( $dtalkid ), 'reply_url' => $reply_url, @@ -496,8 +496,8 @@ sub EntryPage_entry # load the userpic; include the keyword selected by the user # as a backup for the alttext - my $pickw = LJ::Entry->userpic_kw_from_props($entry->props); - my $userpic = Image_userpic($pu, $entry->userpic ? $entry->userpic->picid : 0, $pickw); + my ( $pic, $pickw ) = $entry->userpic; + my $userpic = Image_userpic($pu, $pic ? $pic->picid : 0, $pickw); my $comments = CommentInfo( $entry->comment_info( u => $u, remote => $remote, style_args => $style_args, viewall => $viewall diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/S2/ReplyPage.pm --- a/cgi-bin/LJ/S2/ReplyPage.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/S2/ReplyPage.pm Sat Oct 02 01:27:29 2010 +0800 @@ -118,8 +118,8 @@ sub ReplyPage $comment_values{subject} = $comment->subject_orig; $comment_values{body} = $comment->body_orig; $comment_values{subjecticon} = $comment->prop('subjecticon'); - $comment_values{prop_picture_keyword} = $comment->prop('picture_keyword'); $comment_values{prop_opt_preformatted} = $comment->prop('opt_preformatted'); + $comment_values{prop_picture_keyword} = $comment->userpic_kw; } if ($replytoid) { @@ -130,6 +130,8 @@ sub ReplyPage return; } + # FIXME: Why are we loading the comment manually when we do LJ::Comment->new below + # and could do everything through there. my $sql = "SELECT jtalkid, posterid, state, datepost FROM talk2 ". "WHERE journalid=$u->{'userid'} AND jtalkid=$re_talkid ". "AND nodetype='L' AND nodeid=" . $entry->jitemid; @@ -177,16 +179,17 @@ sub ReplyPage } my $datetime = DateTime_unix(LJ::mysqldate_to_time($parpost->{'datepost'})); - - my ($s2poster, $pu); + my $comment_userpic; - if ($parpost->{'posterid'}) { - $pu = LJ::load_userid($parpost->{'posterid'}); + my $s2poster; + + my $pu = $parentcomment->poster; + if ( $pu ) { return $opts->{handler_return} = 403 if $pu->is_suspended; # do not show comments by suspended users $s2poster = UserLite($pu); - my $pickw = LJ::Entry->userpic_kw_from_props($parpost->{'props'}); - $comment_userpic = Image_userpic($pu, 0, $pickw); + my ( $pic, $pickw ) = $parentcomment->userpic; + $comment_userpic = Image_userpic($pu, $pic ? $pic->picid : 0, $pickw); } LJ::CleanHTML::clean_comment(\$parpost->{'body'}, diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/Talk.pm --- a/cgi-bin/LJ/Talk.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/Talk.pm Sat Oct 02 01:27:29 2010 +0800 @@ -1147,28 +1147,38 @@ sub load_comments } # optionally give them back user refs - if (ref($opts->{'userref'}) eq "HASH") { + if (ref($opts->{userref}) eq "HASH") { my %userpics = (); # copy into their ref the users we've already loaded above. while (my ($k, $v) = each %up) { - $opts->{'userref'}->{$k} = $v; + $opts->{userref}->{$k} = $v; } # optionally load userpics - if (ref($opts->{'userpicref'}) eq "HASH") { + if (ref($opts->{userpicref}) eq "HASH") { my @load_pic; foreach my $talkid (@posts_to_load) { my $post = $posts->{$talkid}; - my $kw; - if ($post->{'props'} && $post->{'props'}->{'picture_keyword'}) { - $kw = $post->{'props'}->{'picture_keyword'}; + my $pu = $opts->{userref}->{$post->{posterid}}; + my ( $id, $kw ); + if ( $pu && $pu->userpic_have_mapid ) { + my $mapid; + if ($post->{props} && $post->{props}->{picture_mapid}) { + $mapid = $post->{props}->{picture_mapid}; + } + $kw = $pu ? $pu->get_keyword_from_mapid( $mapid ) : undef; + $id = $pu ? $pu->get_picid_from_mapid( $mapid ) : undef; + } else { + if ($post->{props} && $post->{props}->{picture_keyword}) { + $kw = $post->{props}->{picture_keyword}; + } + $id = $pu ? $pu->get_picid_from_keyword( $kw ) : undef; } - my $pu = $opts->{'userref'}->{$post->{'posterid'}}; - my $id = $pu ? $pu->get_picid_from_keyword( $kw ) : undef; - $post->{'picid'} = $id; + $post->{picid} = $id; + $post->{pickw} = $kw; push @load_pic, [ $pu, $id ]; } - load_userpics( $opts->{'userpicref'}, \@load_pic ); + load_userpics( $opts->{userpicref}, \@load_pic ); } } return map { $posts->{$_} } @top_replies; @@ -2855,7 +2865,12 @@ sub enter_comment { $talkprop{'unknown8bit'} = 1 if $comment->{unknown8bit}; $talkprop{'subjecticon'} = $comment->{subjecticon}; - $talkprop{'picture_keyword'} = $comment->{picture_keyword}; + my $pu = $comment->{u}; + if ( $pu && $pu->userpic_have_mapid ) { + $talkprop{picture_mapid} = $pu->get_mapid_from_keyword( $comment->{picture_keyword} ); + } else { + $talkprop{picture_keyword} = $comment->{picture_keyword}; + } $talkprop{'opt_preformatted'} = $comment->{preformat} ? 1 : 0; if ($journalu->opt_logcommentips eq "A" || @@ -3015,7 +3030,14 @@ sub enter_imported_comment { $talkprop{'unknown8bit'} = 1 if $comment->{unknown8bit}; $talkprop{'subjecticon'} = $comment->{subjecticon}; - $talkprop{'picture_keyword'} = $comment->{picture_keyword}; + + my $pu = $comment->{u}; + if ( $pu && $pu->userpic_have_mapid ) { + $talkprop{picture_mapid} = $pu->get_mapid_from_keyword( $comment->{picture_keyword}, create => 1 ); + } else { + $talkprop{picture_keyword} = $comment->{picture_keyword}; + } + $talkprop{'opt_preformatted'} = $comment->{preformat} ? 1 : 0; # remove blank/0 values (defaults) @@ -3140,7 +3162,7 @@ sub init { $form->{'userpost'} = $remote->{'user'}; $form->{'usertype'} = "user"; } - # XXXevan hack: remove me when we fix preview. + # FIXME: XXXevan hack: remove me when we fix preview. $init->{cookie_auth} = $cookie_auth; # test accounts may only comment on other test accounts. @@ -3665,10 +3687,16 @@ sub edit_comment { my %props = ( subjecticon => $comment->{subjecticon}, - picture_keyword => $comment->{picture_keyword}, opt_preformatted => $comment->{preformat} ? 1 : 0, edit_reason => $comment->{editreason}, ); + + my $pu = $comment_obj->poster; + if ( $pu && $pu->userpic_have_mapid ) { + $props{picture_mapid} = $pu->get_mapid_from_keyword( $comment->{picture_keyword} ); + } else { + $props{picture_keyword} = $comment->{picture_keyword}; + } # set most of the props together $comment_obj->set_props(%props); @@ -3689,7 +3717,7 @@ sub edit_comment { $comment->{talkid} = $comment_obj->jtalkid; # cluster tracking - LJ::mark_user_active($comment_obj->poster, 'comment'); + LJ::mark_user_active($pu, 'comment'); # fire events if ( LJ::is_enabled('esn') ) { diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/User.pm --- a/cgi-bin/LJ/User.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/User.pm Sat Oct 02 01:27:29 2010 +0800 @@ -6082,6 +6082,7 @@ sub activate_userpics { return 1 if $u->is_expunged; my $userid = $u->userid; + my $have_mapid = $u->userpic_have_mapid; # active / inactive lists my @active = (); @@ -6117,33 +6118,45 @@ sub activate_userpics { # query all pickws in logprop2 with jitemid > that value my %count_kw = (); - my $propid = LJ::get_prop("log", "picture_keyword")->{'id'}; + my $propid; + if ( $have_mapid ) { + $propid = LJ::get_prop("log", "picture_mapid")->{id}; + } else { + $propid = LJ::get_prop("log", "picture_keyword")->{id}; + } my $sth = $dbcr->prepare("SELECT value, COUNT(*) FROM logprop2 " . "WHERE journalid=? AND jitemid > ? AND propid=?" . "GROUP BY value"); - $sth->execute($userid, $jitemid, $propid); + $sth->execute($userid, $jitemid || 0, $propid); while (my ($value, $ct) = $sth->fetchrow_array) { # keyword => count $count_kw{$value} = $ct; } - my $keywords_in = join(",", map { $dbh->quote($_) } keys %count_kw); + my $values_in = join(",", map { $dbh->quote($_) } keys %count_kw); # map pickws to picids for freq hash below my %count_picid = (); - if ($keywords_in) { - my $sth = $dbcr->prepare( "SELECT k.keyword, m.picid FROM userkeywords k, userpicmap2 m ". - "WHERE k.keyword IN ($keywords_in) AND k.kwid=m.kwid AND k.userid=m.userid " . - "AND k.userid=?" ); - $sth->execute($userid); - while (my ($keyword, $picid) = $sth->fetchrow_array) { - # keyword => picid - $count_picid{$picid} += $count_kw{$keyword}; + if ( $values_in ) { + if ( $have_mapid ) { + foreach my $mapid ( keys %count_kw ) { + my $picid = $u->get_picid_from_mapid($mapid); + $count_picid{$picid} += $count_kw{$mapid} if $picid; + } + } else { + my $sth = $dbcr->prepare( "SELECT k.keyword, m.picid FROM userkeywords k, userpicmap2 m ". + "WHERE k.keyword IN ($values_in) AND k.kwid=m.kwid AND k.userid=m.userid " . + "AND k.userid=?" ); + $sth->execute($userid); + while (my ($keyword, $picid) = $sth->fetchrow_array) { + # keyword => picid + $count_picid{$picid} += $count_kw{$keyword}; + } } } # we're only going to ban the least used, excluding the user's default - my @ban = (grep { $_ != $u->{'defaultpicid'} } + my @ban = (grep { $_ != $u->{defaultpicid} } sort { $count_picid{$a} <=> $count_picid{$b} } @active); @ban = splice(@ban, 0, $to_ban) if @ban > $to_ban; @@ -6171,6 +6184,7 @@ sub activate_userpics { # delete userpic info object from memcache LJ::Userpic->delete_cache($u); + $u->clear_userpic_kw_map; return 1; } @@ -6221,7 +6235,13 @@ sub expunge_userpic { # else now mark it $u->do( "UPDATE userpic2 SET state='X' WHERE userid = ? AND picid = ?", undef, $u->userid, $picid ); return LJ::error( $dbcm ) if $dbcm->err; + + # Since we don't clean userpicmap2 when we migrate to dversion 9, clean it here on expunge no matter the dversion. $u->do( "DELETE FROM userpicmap2 WHERE userid = ? AND picid = ?", undef, $u->userid, $picid ); + if ( $u->userpic_have_mapid ) { + $u->do( "DELETE FROM userpicmap3 WHERE userid = ? AND picid = ? AND kwid=NULL", undef, $u->userid, $picid ); + $u->do( "UPDATE userpicmap3 SET picid = NULL WHERE userid = ? AND picid = ?", undef, $u->userid, $picid ); + } # now clear the user's memcache picture info LJ::Userpic->delete_cache( $u ); @@ -6231,6 +6251,111 @@ sub expunge_userpic { return ( $u->userid, map {$_->[0]} grep {$_ && @$_ && $_->[0]} @rval ); } +=head3 C<< $u->get_keyword_from_mapid( $mapid, %opts ) >> + +Returns the keyword for the given mapid or undef if the mapid doesn't exist. + +Arguments: + +=over 4 + +=item mapid + +=back + +Additional options: + +=over 4 + +=item redir_callback + +Called if the mapping is redirected to another mapping with the following arguments + +( $u, $old_mapid, $new_mapid ) + +=back + +=cut +sub get_keyword_from_mapid { + my ( $u, $mapid, %opts ) = @_; + my $info = LJ::isu( $u ) ? $u->get_userpic_info : undef; + return undef unless $info; + return undef unless $u->userpic_have_mapid; + + $mapid = $u->resolve_mapid_redirects($mapid,%opts); + my $kw = $info->{mapkw}->{ $mapid }; + return $kw; +} + +=head3 C<< $u->get_mapid_from_keyword( $kw, %opts ) >> + +Returns the mapid for a given keyword. + +Arguments: + +=over 4 + +=item kw + +The keyword. + +=back + +Additional options: + +=over 4 + +=item create + +Should a mapid be created if one does not exist. + +Default: 0 + +=back + +=cut +sub get_mapid_from_keyword { + my ( $u, $kw, %opts ) = @_; + return 0 unless $u->userpic_have_mapid; + + my $info = LJ::isu( $u ) ? $u->get_userpic_info : undef; + return 0 unless $info; + + my $mapid = $info->{kwmap}->{$kw}; + return $mapid if $mapid; + + # the silly "pic#2343" thing when they didn't assign a keyword, if we get here + # we need to create it. + if ( $kw =~ /^pic\#(\d+)$/ ) { + my $picid = $1; + return 0 unless $info->{pic}{$picid}; # don't create rows for invalid pics + return 0 unless $info->{pic}{$picid}{state} eq 'N'; # or inactive + + return $u->_create_mapid( undef, $picid ) + } + + return 0 unless $opts{create}; + + return $u->_create_mapid( $u->get_keyword_id( $kw ), undef ); +} + +=head3 C<< $u->get_picid_from_keyword( $kw, $default ) >> + +Returns the picid for a given keyword. + +=over 4 + +=item kw + +Keyword to look up. + +=item default (optional) + +Default: the users default userpic. + +=back + +=cut sub get_picid_from_keyword { my ( $u, $kw, $default ) = @_; $default ||= ref $u ? $u->{defaultpicid} : 0; @@ -6239,15 +6364,61 @@ sub get_picid_from_keyword { my $info = LJ::isu( $u ) ? $u->get_userpic_info : undef; return $default unless $info; - my $pr = $info->{'kw'}{$kw}; + my $pr = $info->{kw}{$kw}; # normal keyword return $pr->{picid} if $pr->{picid}; # the silly "pic#2343" thing when they didn't assign a keyword if ( $kw =~ /^pic\#(\d+)$/ ) { my $picid = $1; - return $picid if $info->{'pic'}{$picid}; - } + return $picid if $info->{pic}{$picid}; + } + + return $default; +} + +=head3 C<< $u->get_picid_from_mapid( $mapid, %opts ) >> + +Returns the picid for a given mapid. + +Arguments: + +=over 4 + +=item mapid + +=back + +Additional options: + +=over 4 + +=item default + +Default: the users default userpic. + +=item redir_callback + +Called if the mapping is redirected to another mapping with the following arguments + +( $u, $old_mapid, $new_mapid ) + +=back + +=cut +sub get_picid_from_mapid { + my ( $u, $mapid, %opts ) = @_; + my $default = $opts{default} || ref $u ? $u->{defaultpicid} : 0; + return $default unless $mapid; + return $default unless $u->userpic_have_mapid; + + my $info = LJ::isu( $u ) ? $u->get_userpic_info : undef; + return $default unless $info; + + $mapid = $u->resolve_mapid_redirects($mapid,%opts); + my $pr = $info->{mapid}{$mapid}; + + return $pr->{picid} if $pr->{picid}; return $default; } @@ -6340,10 +6511,13 @@ Maps a picid to a pic hashref. # userid, # "packed string", which expands to an array of {width=>..., ...} # "packed string", which expands to { 'kw1' => id, 'kw2' => id, ...} +# series of 3 4-byte numbers, which expands to { mapid1 => id, mapid2 => id, ...}, as well as { mapid1 => mapid2 } +# "packed string", which expands to { 'kw1' => mapid, 'kw2' => mapid, ...} # ] sub get_userpic_info { my ( $u, $opts ) = @_; return undef unless LJ::isu( $u ) && $u->clusterid; + my $mapped_icons = $u->userpic_have_mapid; # in the cache, cool, well unless it doesn't have comments or urls or descriptions # and we need them @@ -6356,7 +6530,7 @@ sub get_userpic_info { return $cachedata if $good; } - my $VERSION_PICINFO = 3; + my $VERSION_PICINFO = 4; my $memkey = [$u->userid,"upicinf:$u->{'userid'}"]; my ($info, $minfo); @@ -6370,13 +6544,13 @@ sub get_userpic_info { # old data in the cache. delete. LJ::MemCache::delete($memkey); } else { - my (undef, $picstr, $kwstr) = @$minfo; + my (undef, $picstr, $kwstr, $picmapstr, $kwmapstr) = @$minfo; $info = { - 'pic' => {}, - 'kw' => {}, + pic => {}, + kw => {} }; while (length $picstr >= 7) { - my $pic = { userid => $u->{'userid'} }; + my $pic = { userid => $u->userid }; ($pic->{picid}, $pic->{width}, $pic->{height}, $pic->{state}) = unpack "NCCA", substr($picstr, 0, 7, ''); @@ -6389,12 +6563,37 @@ sub get_userpic_info { my $kw = substr($kwstr, $pos, $nulpos-$pos); my $id = unpack("N", substr($kwstr, $nulpos+1, 4)); $pos = $nulpos + 5; # skip NUL + 4 bytes. - $info->{kw}->{$kw} = $info->{pic}->{$id} if $info; - } - } + $info->{kw}->{$kw} = $info->{pic}->{$id}; + } + + if ( $mapped_icons ) { + if ( defined $picmapstr && defined $kwmapstr ) { + $pos = 0; + while ($pos < length($picmapstr)) { + my ($mapid, $id, $redir) = unpack("NNN", substr($picmapstr, $pos, 12)); + $pos += 12; # 3 * 4 bytes. + $info->{mapid}->{$mapid} = $info->{pic}{$id} if $id; + $info->{map_redir}->{$mapid} = $redir if $redir; + } + + $pos = $nulpos = 0; + while (($nulpos = index($kwmapstr, "\0", $pos)) > 0) { + my $kw = substr($kwmapstr, $pos, $nulpos-$pos); + my $id = unpack("N", substr($kwmapstr, $nulpos+1, 4)); + $pos = $nulpos + 5; # skip NUL + 4 bytes. + $info->{kwmap}->{$kw} = $id; + $info->{mapkw}->{$id} = $kw || "pic#" . $info->{mapid}->{$id}->{picid}; + } + } else { # This user is on dversion 9, but the data isn't in memcache + # so force a db load + undef $info; + } + } + } + # Load picture comments - if ( $opts->{load_comments} ) { + if ( $opts->{load_comments} && $info ) { my $commemkey = [$u->userid, "upiccom:" . $u->userid]; my $comminfo = LJ::MemCache::get( $commemkey ); @@ -6464,10 +6663,10 @@ sub get_userpic_info { my %minfodesc; unless ($info) { $info = { - 'pic' => {}, - 'kw' => {}, + pic => {}, + kw => {} }; - my ($picstr, $kwstr); + my ($picstr, $kwstr, $predirstr, $kwmapstr); my $sth; my $dbcr = LJ::get_cluster_def_reader($u); my $db = @LJ::MEMCACHE_SERVERS ? LJ::get_db_writer() : LJ::get_db_reader(); @@ -6480,7 +6679,7 @@ sub get_userpic_info { while (my $pic = $sth->fetchrow_hashref) { next if $pic->{state} eq 'X'; # no expunged pics in list push @pics, $pic; - $info->{'pic'}->{$pic->{'picid'}} = $pic; + $info->{pic}->{$pic->{picid}} = $pic; $minfocom{int($pic->{picid})} = $pic->{comment} if $opts->{load_comments} && $pic->{comment}; $minfourl{int($pic->{picid})} = $pic->{url} @@ -6493,20 +6692,47 @@ sub get_userpic_info { $picstr = join('', map { pack("NCCA", $_->{picid}, $_->{width}, $_->{height}, $_->{state}) } @pics); - $sth = $dbcr->prepare( "SELECT k.keyword, m.picid FROM userpicmap2 m, userkeywords k ". - "WHERE k.userid=? AND m.kwid=k.kwid AND m.userid=k.userid" ); + if ( $mapped_icons ) { + $sth = $dbcr->prepare( "SELECT k.keyword, m.picid, m.mapid, m.redirect_mapid FROM userpicmap3 m LEFT JOIN userkeywords k ON ". + "( m.userid=k.userid AND m.kwid=k.kwid ) WHERE m.userid=?" ); + } else { + $sth = $dbcr->prepare( "SELECT k.keyword, m.picid FROM userpicmap2 m, userkeywords k ". + "WHERE k.userid=? AND m.kwid=k.kwid AND m.userid=k.userid" ); + } $sth->execute($u->{'userid'}); my %minfokw; - while (my ($kw, $id) = $sth->fetchrow_array) { - next unless $info->{'pic'}->{$id}; + my %picmap; + my %kwmap; + while (my ($kw, $id, $mapid, $redir) = $sth->fetchrow_array) { + my $skip_kw = 0; + if ( $mapped_icons ) { + $picmap{$mapid} = [ int($id), int($redir) ]; + if ( $redir ) { + $info->{map_redir}->{$mapid} = $redir; + } else { + unless ( defined $kw ) { + $skip_kw = 1; + $kw = "pic#$id"; + } + $info->{kwmap}->{$kw} = $kwmap{$kw} = $mapid; + $info->{mapkw}->{$mapid} = $kw; + } + } + next if $skip_kw; + next unless $info->{pic}->{$id}; next if $kw =~ /[\n\r\0]/; # used to be a bug that allowed these to get in. - $info->{'kw'}->{$kw} = $info->{'pic'}->{$id}; + $info->{kw}->{$kw} = $info->{pic}->{$id}; + $info->{mapid}->{$mapid} = $info->{pic}->{$id} if $mapped_icons && $id; $minfokw{$kw} = int($id); } $kwstr = join('', map { pack("Z*N", $_, $minfokw{$_}) } keys %minfokw); + if ( $mapped_icons ) { + $predirstr = join('', map { pack("NNN", $_, @{ $picmap{$_} } ) } keys %picmap); + $kwmapstr = join('', map { pack("Z*N", $_, $kwmap{$_}) } keys %kwmap); + } $memkey = [$u->{'userid'},"upicinf:$u->{'userid'}"]; - $minfo = [ $VERSION_PICINFO, $picstr, $kwstr ]; + $minfo = [ $VERSION_PICINFO, $picstr, $kwstr, $predirstr, $kwmapstr ]; LJ::MemCache::set($memkey, $minfo); if ( $opts->{load_comments} ) { @@ -6558,7 +6784,7 @@ sub get_userpic_kw_map { foreach my $keyword ( keys %{$picinfo->{kw}} ) { my $picid = $picinfo->{kw}->{$keyword}->{picid}; $keywords->{$picid} = [] unless $keywords->{$picid}; - push @{$keywords->{$picid}}, $keyword if ( $keyword && $picid ); + push @{$keywords->{$picid}}, $keyword if ( $keyword && $picid && $keyword !~ m/^pic\#(\d+)$/ ); } return $u->{picid_kw_map} = $keywords; @@ -6587,6 +6813,59 @@ sub mogfs_userpic_key { return "up:" . $self->userid . ":$picid"; } +=head3 C<< $u->resolve_mapid_redirects( $mapid, %opts ) >> + +Resolve any mapid redirect, guarding against any redirect loops. + +Returns: new map id, or 0 if the mapping cannot be resolved. + +Arguments: + +=over 4 + +=item mapid + +=back + +Additional options: + +=over 4 + +=item redir_callback + +Called if the mapping is redirected to another mapping with the following arguments + +( $u, $old_mapid, $new_mapid ) + +=back + +=cut +sub resolve_mapid_redirects { + my ( $u, $mapid, %opts ) = @_; + + my $info = LJ::isu( $u ) ? $u->get_userpic_info : undef; + return 0 unless $info; + + my %seen = ( $mapid => 1 ); + my $orig_id = $mapid; + + while ( $info->{map_redir}->{ $mapid } ) { + $orig_id = $mapid; + $mapid = $info->{map_redir}->{ $mapid }; + + # To implement lazy updating or the like + $opts{redir_callback}->($u, $orig_id, $mapid) if $opts{redir_callback}; + + # This should never happen, but am checking it here mainly in case + # never *does* happen, so we don't hang the web process with an endless loop. + if ( $seen{$mapid}++ ) { + warn("userpicmap3 redirectloop for " . $u->id . " on mapid " . $mapid); + return 0; + } + } + + return $mapid; +} =head3 C<< $u->userpic >> @@ -6599,6 +6878,15 @@ sub userpic { return LJ::Userpic->new($u, $u->{defaultpicid}); } +=head3 C<< $u->userpic_have_mapid >> + +Returns true if the userpicmap keyword mappings have a mapid column ( dversion 9 or higher ) + +=cut +# FIXME: This probably should be userpics_use_mapid +sub userpic_have_mapid { + return $_[0]->dversion >= 9; +} =head3 C<< $u->userpic_quota >> @@ -6613,7 +6901,23 @@ sub userpic_quota { return $quota; } - +# Intentionally no POD here. +# This is an internal helper method +# takes a $kwid and $picid ( either can be undef ) +# and creates a mapid row for it +sub _create_mapid { + my ( $u, $kwid, $picid ) = @_; + return 0 unless $u->userpic_have_mapid; + + my $mapid = LJ::alloc_user_counter($u,'Y'); + $u->do( "INSERT INTO userpicmap3 (userid, mapid, kwid, picid) VALUES (?,?,?,?)", undef, $u->id, $mapid, $kwid, $picid); + return 0 if $u->err; + + LJ::Userpic->delete_cache($u); + $u->clear_userpic_kw_map; + + return $mapid; +} ######################################################################## ### 99. Miscellaneous Legacy Items @@ -7390,7 +7694,7 @@ sub unset_remote # 'Q' == Notification Inbox, # 'D' == 'moDule embed contents', 'I' == Import data block # 'Z' == import status item, 'X' == eXternal account -# 'F' == filter id +# 'F' == filter id, 'Y' = pic/keYword mapping id # sub alloc_user_counter { @@ -7399,7 +7703,7 @@ sub alloc_user_counter ################################################################## # IF YOU UPDATE THIS MAKE SURE YOU ADD INITIALIZATION CODE BELOW # - return undef unless $dom =~ /^[LTMPSRKCOVEQGDIZXF]$/; # + return undef unless $dom =~ /^[LTMPSRKCOVEQGDIZXFY]$/; # ################################################################## my $dbh = LJ::get_db_writer(); @@ -7520,6 +7824,9 @@ sub alloc_user_counter } elsif ($dom eq "F") { $newmax = $u->selectrow_array("SELECT MAX(filterid) FROM watch_filters WHERE userid=?", undef, $uid); + } elsif ($dom eq "Y") { + $newmax = $u->selectrow_array("SELECT MAX(mapid) FROM userpicmap3 WHERE userid=?", + undef, $uid); } else { die "No user counter initializer defined for area '$dom'.\n"; } diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/LJ/Userpic.pm --- a/cgi-bin/LJ/Userpic.pm Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/LJ/Userpic.pm Sat Oct 02 01:27:29 2010 +0800 @@ -60,6 +60,14 @@ sub reset_singletons { sub reset_singletons { %singletons = (); } + +=head1 NAME + +LJ::Userpic + +=head1 Class Methods + +=cut # LJ::Userpic constructor. Returns a LJ::Userpic object. # Return existing with userid and picid populated, or make new. @@ -148,8 +156,12 @@ sub new_from_row { return $self; } -sub new_from_keyword -{ +=head2 C<< $class->new_from_keyword( $u, $kw ) >> + +Returns the LJ::Userpic for the given keyword + +=cut +sub new_from_keyword { my ( $class, $u, $kw ) = @_; return undef unless LJ::isu( $u ); @@ -158,7 +170,24 @@ sub new_from_keyword return $picid ? $class->new( $u, $picid ) : undef; } -# instance methods + +=head2 C<< $class->new_from_mapid( $u, $mapid ) >> + +Returns the LJ::Userpic for the given mapid + +=cut +sub new_from_mapid { + my ( $class, $u, $mapid ) = @_; + return undef unless LJ::isu( $u ); + + my $picid = $u->get_picid_from_mapid( $mapid ); + + return $picid ? $class->new( $u, $picid ) : undef; +} + +=head1 Instance Methods + +=cut sub valid { return defined $_[0]->state; @@ -934,11 +963,14 @@ sub delete { # userpic keywords eval { - $u->do( "DELETE FROM userpicmap2 WHERE userid=? " . - "AND picid=?", undef, $u->userid, $picid ) or die; - $u->do( "DELETE FROM userpic2 WHERE picid=? AND userid=?", - undef, $picid, $u->userid ) or die; - }; + if ( $u->userpic_have_mapid ) { + $u->do( "DELETE FROM userpicmap3 WHERE userid = ? AND picid = ? AND kwid=NULL", undef, $u->userid, $picid ) or die; + $u->do( "UPDATE userpicmap3 SET picid=NULL WHERE userid=? AND picid=?", undef, $u->userid, $picid ) or die; + } else { + $u->do( "DELETE FROM userpicmap2 WHERE userid=? AND picid=?", undef, $u->userid, $picid ) or die; + } + $u->do( "DELETE FROM userpic2 WHERE picid=? AND userid=?", undef, $picid, $u->userid ) or die; + }; $fail->() if $@; $u->log_event('delete_userpic', { picid => $picid }); @@ -1005,15 +1037,31 @@ sub set_keywords { @keywords = grep { !/^pic\#\d+$/ } grep { s/^\s+//; s/\s+$//; $_; } @keywords; my $u = $self->owner; + my $have_mapid = $u->userpic_have_mapid; + my $sth; my $dbh; - $sth = $u->prepare( "SELECT kwid FROM userpicmap2 WHERE userid=? AND picid=?" ); + if ( $have_mapid ) { + $sth = $u->prepare( "SELECT kwid FROM userpicmap3 WHERE userid=? AND picid=?" ); + } else { + $sth = $u->prepare( "SELECT kwid FROM userpicmap2 WHERE userid=? AND picid=?" ); + } $sth->execute( $u->userid, $self->id ); my %exist_kwids; while (my ($kwid) = $sth->fetchrow_array) { $exist_kwids{$kwid} = 1; + } + + my %kwid_to_mapid; + if ( $have_mapid ) { + $sth = $u->prepare( "SELECT mapid, kwid FROM userpicmap3 WHERE userid=?" ); + $sth->execute( $u->userid ); + + while (my ($mapid, $kwid) = $sth->fetchrow_array) { + $kwid_to_mapid{$kwid} = $mapid; + } } my (@bind, @data, @kw_errors); @@ -1030,23 +1078,39 @@ sub set_keywords { } unless (delete $exist_kwids{$kwid}) { - push @bind, '(?, ?, ?)'; - push @data, $u->{'userid'}, $kwid, $picid; + if ( $have_mapid ) { + $kwid_to_mapid{$kwid} ||= LJ::alloc_user_counter( $u, 'Y' ); + + push @bind, '(?, ?, ?, ?)'; + push @data, $u->userid, $kwid_to_mapid{$kwid}, $kwid, $picid; + } else { + push @bind, '(?, ?, ?)'; + push @data, $u->userid, $kwid, $picid; + } } } LJ::Userpic->delete_cache($u); foreach my $kwid (keys %exist_kwids) { - $u->do("DELETE FROM userpicmap2 WHERE userid=? AND picid=? AND kwid=?", undef, $u->{userid}, $self->id, $kwid); + if ( $have_mapid ) { + $u->do("UPDATE userpicmap3 SET picid=NULL WHERE userid=? AND picid=? AND kwid=?", undef, $u->id, $self->id, $kwid); + } else { + $u->do("DELETE FROM userpicmap2 WHERE userid=? AND picid=? AND kwid=?", undef, $u->id, $self->id, $kwid); + } } # save data if any if (scalar @data) { my $bind = join(',', @bind); - $u->do( "REPLACE INTO userpicmap2 (userid, kwid, picid) VALUES $bind", - undef, @data ); + if ( $have_mapid ) { + $u->do( "REPLACE INTO userpicmap3 (userid, mapid, kwid, picid) VALUES $bind", + undef, @data ); + } else { + $u->do( "REPLACE INTO userpicmap2 (userid, kwid, picid) VALUES $bind", + undef, @data ); + } } # clear the userpic-keyword map. diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/ljlib.pl --- a/cgi-bin/ljlib.pl Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/ljlib.pl Sat Oct 02 01:27:29 2010 +0800 @@ -126,7 +126,7 @@ sub END { LJ::end_request(); } "embedcontent", "usermsg", "usermsgtext", "usermsgprop", "notifyarchive", "notifybookmarks", "pollprop2", "embedcontent_preview", "logprop_history", "import_status", "externalaccount", - "content_filters", "content_filter_data", + "content_filters", "content_filter_data", "userpicmap3", ); # keep track of what db locks we have out diff -r ebea5b13e8a4 -r f897918203ac cgi-bin/ljprotocol.pl --- a/cgi-bin/ljprotocol.pl Sat Oct 02 00:29:19 2010 +0800 +++ b/cgi-bin/ljprotocol.pl Sat Oct 02 01:27:29 2010 +0800 @@ -1132,6 +1132,7 @@ sub postevent $flags->{noauth} = 1; $flags->{usejournal_okay} = 1; $flags->{no_xpost} = 1; + $flags->{create_unknown_picture_mapid} = 1; } return undef unless LJ::Hooks::run_hook('post_noauth', $req) || authenticate($req, $err, $flags); @@ -1221,6 +1222,12 @@ sub postevent return undef unless common_event_validation($req, $err, $flags); + + # now we can move over to picture_mapid instead of picture_keyword if appropriate + if ( $req->{props} && $req->{props}->{picture_keyword} && $u->userpic_have_mapid ) { + $req->{props}->{picture_mapid} = $u->get_mapid_from_keyword( $req->{props}->{picture_keyword}, create => $flags->{create_unknown_picture_mapid} || 0 ); + delete $req->{props}->{picture_keyword}; + } # confirm we can add tags, at least return fail($err, 312) @@ -1886,6 +1893,12 @@ sub editevent return undef unless common_event_validation($req, $err, $flags); + # now we can move over to picture_mapid instead of picture_keyword if appropriate + if ( $req->{props} && $req->{props}->{picture_keyword} && $u->userpic_have_mapid ) { + $req->{props}->{picture_mapid} = $u->get_mapid_from_keyword( $req->{props}->{picture_keyword}, create => $flags->{create_unknown_picture_mapid} || 0 ); + delete $req->{props}->{picture_keyword}; + } + ## handle meta-data (properties) my %props_byname = (); foreach my $key (keys %{$req->{'props'}}) { @@ -2409,10 +2422,15 @@ sub getevents # if they want subjects to be events, replace event # with subject when requested. - if ($req->{'prefersubject'} && length($t->[0])) { + if ($req->{prefersubject} && length($t->[0])) { $t->[1] = $t->[0]; # event = subject $t->[0] = undef; # subject = undef } + + # re-generate the picture_keyword prop for the returned data, as a mapid will mean nothing + my $pu = $uowner; + $pu = LJ::load_user( $evt->{poster} ) if $evt->{poster}; + $evt->{props}->{picture_keyword} = $pu->get_keyword_from_mapid( $evt->{props}->{picture_mapid} ) if $pu->userpic_have_mapid; # now that we have the subject, the event and the props, # auto-translate them to UTF-8 if they're not in UTF-8. @@ -3068,17 +3086,18 @@ sub list_pickws my %seen; # mashifiedptr -> 1 # FIXME: should be a utf-8 sort - foreach my $kw (sort keys %{$pi->{'kw'}}) { - my $pic = $pi->{'kw'}{$kw}; + foreach my $kw ( sort keys %{$pi->{kw}} ) { + my $pic = $pi->{kw}{$kw}; $seen{$pic} = 1; - next if $pic->{'state'} eq "I"; - push @res, [ $kw, $pic->{'picid'} ]; + next if $pic->{state} eq "I"; + push @res, [ $kw, $pic->{picid} ]; } # now add all the pictures that don't have a keyword - foreach my $picid (keys %{$pi->{'pic'}}) { - my $pic = $pi->{'pic'}{$picid}; + foreach my $picid ( keys %{$pi->{pic}} ) { + my $pic = $pi->{pic}{$picid}; next if $seen{$pic}; + next if $pic->{state} eq "I"; push @res, [ "pic#$picid", $picid ]; } @@ -3852,6 +3871,7 @@ sub editevent $res->{'itemid'} = $rs->{'itemid'}; $res->{'anum'} = $rs->{'anum'} if defined $rs->{'anum'}; $res->{'url'} = $rs->{'url'} if defined $rs->{'url'}; + return 1; } diff -r ebea5b13e8a4 -r f897918203ac htdocs/admin/entryprops.bml --- a/htdocs/admin/entryprops.bml Sat Oct 02 00:29:19 2010 +0800 +++ b/htdocs/admin/entryprops.bml Sat Oct 02 01:27:29 2010 +0800 @@ -59,8 +59,10 @@ body<= } } + my $pu = $entry->poster; + $ret .= "<strong>Subject</strong>: <a href=" . $entry->url . ">" . $subject . "</a><br />"; - $ret .= "<strong>Poster</strong>: " . $entry->poster->ljuser_display . "<br />"; + $ret .= "<strong>Poster</strong>: " . $pu->ljuser_display . "<br />"; $ret .= "<strong>Journal</strong>: " . $entry->journal->ljuser_display . "<br />"; $ret .= "<strong>Security</strong>: " . $security . " "; $ret .= "(journal wide minsecurity: " . ($entry->journal->prop("newpost_minsecurity") || "public") . ")<br />"; @@ -82,7 +84,7 @@ body<= # render xpost prop into human readable form if ( $prop eq "xpost" || $prop eq "xpostdetail" ) { - my %external_accounts_map = map { $_->acctid => $_->servername } DW::External::Account->get_external_accounts( $entry->poster ); + my %external_accounts_map = map { $_->acctid => $_->servername } DW::External::Account->get_external_accounts( $pu ); # FIXME: temporary; trying to figure out when this is undef my $xpost_prop = $props{$prop}; @@ -100,6 +102,20 @@ body<= # FIXME: temporary $props{$prop} .= "raw information about $prop - <input type='text' value='$xpost_prop' />" unless $xpost_hash; + } elsif ( $prop eq 'picture_mapid' && $pu->userpic_have_mapid ) { + my $result = "$props{$prop} -> "; + my $kw = $pu->get_keyword_from_mapid( $props{$prop}, + redir_callback => sub { + $result .= "$_[2] -> "; + }); + $result .= $kw; + my $picid = $pu->get_picid_from_keyword($kw,-1); + if ( $picid == -1 ) { + $result .= " ( not assigned to an icon )"; + } else { + $result .= " ( assigned to an icon )"; + } + $props{$prop} = $result; } $extra = "<br /><small>$p->{des}</small>"; diff -r ebea5b13e8a4 -r f897918203ac htdocs/talkpost.bml --- a/htdocs/talkpost.bml Sat Oct 02 00:29:19 2010 +0800 +++ b/htdocs/talkpost.bml Sat Oct 02 01:27:29 2010 +0800 @@ -53,8 +53,9 @@ body<= $FORM{subject} = $comment->subject_orig; $FORM{body} = $comment->body_orig; $FORM{subjecticon} = $comment->prop('subjecticon'); - $FORM{prop_picture_keyword} = $comment->prop('picture_keyword'); $FORM{prop_opt_preformatted} = $comment->prop('opt_preformatted'); + + $FORM{prop_picture_keyword} = $comment->userpic_kw; } if ($uri =~ m!/(\d+)\.html$!) { diff -r ebea5b13e8a4 -r f897918203ac htdocs/talkread.bml --- a/htdocs/talkread.bml Sat Oct 02 00:29:19 2010 +0800 +++ b/htdocs/talkread.bml Sat Oct 02 01:27:29 2010 +0800 @@ -239,7 +239,7 @@ body<= $ret .= "<p>"; $ret .= "<table id='poster'><tr>"; - my $userpic = $entry->userpic; + my ( $userpic, $kw ) = $entry->userpic; LJ::Hooks::run_hook('notify_event_displayed', $entry); # Build the userpic image tag for the entry's userpic @@ -249,7 +249,7 @@ body<= my $apost = "</a></td>"; # for each image, get the html imgtag - $ret .= $apre . $userpic->imgtag( keyword => $props->{picture_keyword}, user => $up ) . $apost; + $ret .= $apre . $userpic->imgtag( keyword => $kw, user => $up ) . $apost; } $ret .= "<td class='attrib' valign='bottom'>"; @@ -498,7 +498,7 @@ body<= # get the picture keyword from the comment properties # get the comment poster as well my %kwopts = ( - keyword => $post->{props}->{picture_keyword}, + keyword => $post->{pickw}, user => $upost, ); --------------------------------------------------------------------------------