<!DOCTYPE html PUBLIC "-//W3C//DTD XHTML 1.1//EN"
"http://www.w3.org/TR/xhtml11/DTD/xhtml11.dtd">
<html xmlns="http://www.w3.org/1999/xhtml">
<head><meta http-equiv="content-type" content="text/html; charset=utf-8" />
<title>[25313] trunk: Replace the ancient `phpfreaks.com` RegEx to extract urls to ping with a more robust matcher.</title>
</head>
<body>
<style type="text/css"><!--
#msg dl.meta { border: 1px #006 solid; background: #369; padding: 6px; color: #fff; }
#msg dl.meta dt { float: left; width: 6em; font-weight: bold; }
#msg dt:after { content:':';}
#msg dl, #msg dt, #msg ul, #msg li, #header, #footer, #logmsg { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; }
#msg dl a { font-weight: bold}
#msg dl a:link { color:#fc3; }
#msg dl a:active { color:#ff0; }
#msg dl a:visited { color:#cc6; }
h3 { font-family: verdana,arial,helvetica,sans-serif; font-size: 10pt; font-weight: bold; }
#msg pre { overflow: auto; background: #ffc; border: 1px #fa0 solid; padding: 6px; }
#logmsg { background: #ffc; border: 1px #fa0 solid; padding: 1em 1em 0 1em; }
#logmsg p, #logmsg pre, #logmsg blockquote { margin: 0 0 1em 0; }
#logmsg p, #logmsg li, #logmsg dt, #logmsg dd { line-height: 14pt; }
#logmsg h1, #logmsg h2, #logmsg h3, #logmsg h4, #logmsg h5, #logmsg h6 { margin: .5em 0; }
#logmsg h1:first-child, #logmsg h2:first-child, #logmsg h3:first-child, #logmsg h4:first-child, #logmsg h5:first-child, #logmsg h6:first-child { margin-top: 0; }
#logmsg ul, #logmsg ol { padding: 0; list-style-position: inside; margin: 0 0 0 1em; }
#logmsg ul { text-indent: -1em; padding-left: 1em; }#logmsg ol { text-indent: -1.5em; padding-left: 1.5em; }
#logmsg > ul, #logmsg > ol { margin: 0 0 1em 0; }
#logmsg pre { background: #eee; padding: 1em; }
#logmsg blockquote { border: 1px solid #fa0; border-left-width: 10px; padding: 1em 1em 0 1em; background: white;}
#logmsg dl { margin: 0; }
#logmsg dt { font-weight: bold; }
#logmsg dd { margin: 0; padding: 0 0 0.5em 0; }
#logmsg dd:before { content:'\00bb';}
#logmsg table { border-spacing: 0px; border-collapse: collapse; border-top: 4px solid #fa0; border-bottom: 1px solid #fa0; background: #fff; }
#logmsg table th { text-align: left; font-weight: normal; padding: 0.2em 0.5em; border-top: 1px dotted #fa0; }
#logmsg table td { text-align: right; border-top: 1px dotted #fa0; padding: 0.2em 0.5em; }
#logmsg table thead th { text-align: center; border-bottom: 1px solid #fa0; }
#logmsg table th.Corner { text-align: left; }
#logmsg hr { border: none 0; border-top: 2px dashed #fa0; height: 1px; }
#header, #footer { color: #fff; background: #636; border: 1px #300 solid; padding: 6px; }
#patch { width: 100%; }
#patch h4 {font-family: verdana,arial,helvetica,sans-serif;font-size:10pt;padding:8px;background:#369;color:#fff;margin:0;}
#patch .propset h4, #patch .binary h4 {margin:0;}
#patch pre {padding:0;line-height:1.2em;margin:0;}
#patch .diff {width:100%;background:#eee;padding: 0 0 10px 0;overflow:auto;}
#patch .propset .diff, #patch .binary .diff {padding:10px 0;}
#patch span {display:block;padding:0 10px;}
#patch .modfile, #patch .addfile, #patch .delfile, #patch .propset, #patch .binary, #patch .copfile {border:1px solid #ccc;margin:10px 0;}
#patch ins {background:#dfd;text-decoration:none;display:block;padding:0 10px;}
#patch del {background:#fdd;text-decoration:none;display:block;padding:0 10px;}
#patch .lines, .info {color:#888;background:#fff;}
--></style>
<div id="msg">
<dl class="meta">
<dt>Revision</dt> <dd><a href="http://core.trac.wordpress.org/changeset/25313">25313</a></dd>
<dt>Author</dt> <dd>wonderboymusic</dd>
<dt>Date</dt> <dd>2013-09-10 03:17:51 +0000 (Tue, 10 Sep 2013)</dd>
</dl>
<h3>Log Message</h3>
<pre>Replace the ancient `phpfreaks.com` RegEx to extract urls to ping with a more robust matcher. URLs with commas and things like `&` were not being pinged. The new matcher even works for most IDN URLs. Adds unit tests.
Fixes <a href="http://core.trac.wordpress.org/ticket/9064">#9064</a>.</pre>
<h3>Modified Paths</h3>
<ul>
<li><a href="#trunksrcwpincludescommentphp">trunk/src/wp-includes/comment.php</a></li>
<li><a href="#trunksrcwpincludesfunctionsphp">trunk/src/wp-includes/functions.php</a></li>
<li><a href="#trunktestsphpunittestsfunctionsphp">trunk/tests/phpunit/tests/functions.php</a></li>
</ul>
</div>
<div id="patch">
<h3>Diff</h3>
<a id="trunksrcwpincludescommentphp"></a>
<div class="modfile"><h4>Modified: trunk/src/wp-includes/comment.php (25312 => 25313)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/src/wp-includes/comment.php 2013-09-10 03:01:10 UTC (rev 25312)
+++ trunk/src/wp-includes/comment.php 2013-09-10 03:17:51 UTC (rev 25313)
</span><span class="lines">@@ -1827,17 +1827,9 @@
</span><span class="cx">
</span><span class="cx"> $pung = get_pung($post_ID);
</span><span class="cx">
</span><del>- // Variables
- $ltrs = '\w';
- $gunk = '/#~:.?+=&%@!\-';
- $punc = '.:?\-';
- $any = $ltrs . $gunk . $punc;
-
</del><span class="cx"> // Step 1
</span><span class="cx"> // Parsing the post, external links (if any) are stored in the $post_links array
</span><del>- // This regexp comes straight from phpfreaks.com
- // http://www.phpfreaks.com/quickcode/Extract_All_URLs_on_a_Page/15.php
- preg_match_all("{\b http : [$any] +? (?= [$punc] * [^$any] | $)}x", $content, $post_links_temp);
</del><ins>+ $post_links_temp = wp_extract_urls( $content );
</ins><span class="cx">
</span><span class="cx"> // Step 2.
</span><span class="cx"> // Walking thru the links array
</span><span class="lines">@@ -1848,7 +1840,7 @@
</span><span class="cx"> // http://dummy-weblog.org/post.php
</span><span class="cx"> // We don't wanna ping first and second types, even if they have a valid <link/>
</span><span class="cx">
</span><del>- foreach ( (array) $post_links_temp[0] as $link_test ) :
</del><ins>+ foreach ( (array) $post_links_temp as $link_test ) :
</ins><span class="cx"> if ( !in_array($link_test, $pung) && (url_to_postid($link_test) != $post_ID) // If we haven't pung it already and it isn't a link to itself
</span><span class="cx"> && !is_local_attachment($link_test) ) : // Also, let's never ping local attachments.
</span><span class="cx"> if ( $test = @parse_url($link_test) ) {
</span></span></pre></div>
<a id="trunksrcwpincludesfunctionsphp"></a>
<div class="modfile"><h4>Modified: trunk/src/wp-includes/functions.php (25312 => 25313)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/src/wp-includes/functions.php 2013-09-10 03:01:10 UTC (rev 25312)
+++ trunk/src/wp-includes/functions.php 2013-09-10 03:17:51 UTC (rev 25313)
</span><span class="lines">@@ -393,6 +393,26 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> /**
</span><ins>+ * Use RegEx to extract URLs from arbitrary content
+ *
+ * @since 3.7.0
+ *
+ * @param string $content
+ * @return array URLs found in passed string
+ */
+function wp_extract_urls( $content ) {
+ preg_match_all(
+ "#((?:[\w-]+://?|[\w\d]+[.])[^\s()<>]+[.](?:\([\w\d]+\)|(?:[^`!()\[\]{};:'\".,<>?«»“”‘’\s]|(?:[:]\d+)?/?)+))#",
+ $content,
+ $post_links
+ );
+
+ $post_links = array_unique( array_map( 'html_entity_decode', $post_links[0] ) );
+
+ return array_values( $post_links );
+}
+
+/**
</ins><span class="cx"> * Check content for video and audio links to add as enclosures.
</span><span class="cx"> *
</span><span class="cx"> * Will not add enclosures that have already been added and will
</span><span class="lines">@@ -417,22 +437,17 @@
</span><span class="cx">
</span><span class="cx"> $pung = get_enclosed( $post_ID );
</span><span class="cx">
</span><del>- $ltrs = '\w';
- $gunk = '/#~:.?+=&%@!\-';
- $punc = '.:?\-';
- $any = $ltrs . $gunk . $punc;
</del><ins>+ $post_links_temp = wp_extract_urls( $content );
</ins><span class="cx">
</span><del>- preg_match_all( "{\b https? : [$any] +? (?= [$punc] * [^$any] | $)}x", $content, $post_links_temp );
-
</del><span class="cx"> foreach ( $pung as $link_test ) {
</span><del>- if ( !in_array( $link_test, $post_links_temp[0] ) ) { // link no longer in post
</del><ins>+ if ( ! in_array( $link_test, $post_links_temp ) ) { // link no longer in post
</ins><span class="cx"> $mids = $wpdb->get_col( $wpdb->prepare("SELECT meta_id FROM $wpdb->postmeta WHERE post_id = %d AND meta_key = 'enclosure' AND meta_value LIKE (%s)", $post_ID, like_escape( $link_test ) . '%') );
</span><span class="cx"> foreach ( $mids as $mid )
</span><span class="cx"> delete_metadata_by_mid( 'post', $mid );
</span><span class="cx"> }
</span><span class="cx"> }
</span><span class="cx">
</span><del>- foreach ( (array) $post_links_temp[0] as $link_test ) {
</del><ins>+ foreach ( (array) $post_links_temp as $link_test ) {
</ins><span class="cx"> if ( !in_array( $link_test, $pung ) ) { // If we haven't pung it already
</span><span class="cx"> $test = @parse_url( $link_test );
</span><span class="cx"> if ( false === $test )
</span></span></pre></div>
<a id="trunktestsphpunittestsfunctionsphp"></a>
<div class="modfile"><h4>Modified: trunk/tests/phpunit/tests/functions.php (25312 => 25313)</h4>
<pre class="diff"><span>
<span class="info">--- trunk/tests/phpunit/tests/functions.php 2013-09-10 03:01:10 UTC (rev 25312)
+++ trunk/tests/phpunit/tests/functions.php 2013-09-10 03:17:51 UTC (rev 25313)
</span><span class="lines">@@ -99,9 +99,9 @@
</span><span class="cx"> }
</span><span class="cx">
</span><span class="cx"> function test_wp_unique_filename() {
</span><del>-
</del><ins>+
</ins><span class="cx"> $testdir = DIR_TESTDATA . '/images/';
</span><del>-
</del><ins>+
</ins><span class="cx"> // sanity check
</span><span class="cx"> $this->assertEquals( 'abcdefg.png', wp_unique_filename( $testdir, 'abcdefg.png' ), 'Sanitiy check failed' );
</span><span class="cx">
</span><span class="lines">@@ -112,10 +112,10 @@
</span><span class="cx">
</span><span class="cx"> // check special chars
</span><span class="cx"> $this->assertEquals( 'testtést-imagé.png', wp_unique_filename( $testdir, 'testtést-imagé.png' ), 'Filename with special chars failed' );
</span><del>-
</del><ins>+
</ins><span class="cx"> // check special chars with potential conflicting name
</span><span class="cx"> $this->assertEquals( 'tést-imagé.png', wp_unique_filename( $testdir, 'tést-imagé.png' ), 'Filename with special chars failed' );
</span><del>-
</del><ins>+
</ins><span class="cx"> // check with single quotes in name (somehow)
</span><span class="cx"> $this->assertEquals( "abcdefgh.png", wp_unique_filename( $testdir, "abcdefg'h.png" ), 'File with quote failed' );
</span><span class="cx">
</span><span class="lines">@@ -124,7 +124,7 @@
</span><span class="cx">
</span><span class="cx"> // test crazy name (useful for regression tests)
</span><span class="cx"> $this->assertEquals( '12%af34567890@..%^_+qwerty-fghjkl-zx.png', wp_unique_filename( $testdir, '12%af34567890#~!@#$..%^&*()|_+qwerty fgh`jkl zx<>?:"{}[]="\'/?.png' ), 'Failed crazy file name' );
</span><del>-
</del><ins>+
</ins><span class="cx"> // test slashes in names
</span><span class="cx"> $this->assertEquals( 'abcdefg.png', wp_unique_filename( $testdir, 'abcde\fg.png' ), 'Slash not removed' );
</span><span class="cx"> $this->assertEquals( 'abcdefg.png', wp_unique_filename( $testdir, 'abcde\\fg.png' ), 'Double slashed not removed' );
</span><span class="lines">@@ -362,4 +362,161 @@
</span><span class="cx"> function test_data_is_not_an_allowed_protocol() {
</span><span class="cx"> $this->assertNotContains( 'data', wp_allowed_protocols() );
</span><span class="cx"> }
</span><ins>+
+ /**
+ * @ticket 9064
+ */
+ function test_wp_extract_urls() {
+ $original_urls = array(
+ 'http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html',
+ 'http://this.com',
+ 'http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437',
+ 'http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html',
+ 'http://wordpress-core.com:8080/',
+ 'http://www.website.com:5000',
+ 'http://wordpress-core/?346236346326&2134362574863.437',
+ 'http://افغانستا.icom.museum',
+ 'http://الجزائر.icom.museum',
+ 'http://österreich.icom.museum',
+ 'http://বাংলাদেশ.icom.museum',
+ 'http://беларусь.icom.museum',
+ 'http://belgië.icom.museum',
+ 'http://българия.icom.museum',
+ 'http://تشادر.icom.museum',
+ 'http://中国.icom.museum',
+ #'http://القمر.icom.museum', // Comoros http://القمر.icom.museum
+ #'http://κυπρος.icom.museum', Cyprus http://κυπρος.icom.museum
+ 'http://českárepublika.icom.museum',
+ #'http://مصر.icom.museum', // Egypt http://مصر.icom.museum
+ 'http://ελλάδα.icom.museum',
+ 'http://magyarország.icom.museum',
+ 'http://ísland.icom.museum',
+ 'http://भारत.icom.museum',
+ 'http://ايران.icom.museum',
+ 'http://éire.icom.museum',
+ 'http://איקו״ם.ישראל.museum',
+ 'http://日本.icom.museum',
+ 'http://الأردن.icom.museum',
+ 'http://қазақстан.icom.museum',
+ 'http://한국.icom.museum',
+ 'http://кыргызстан.icom.museum',
+ 'http://ລາວ.icom.museum',
+ 'http://لبنان.icom.museum',
+ 'http://македонија.icom.museum',
+ #'http://موريتانيا.icom.museum', // Mauritania http://موريتانيا.icom.museum
+ 'http://méxico.icom.museum',
+ 'http://монголулс.icom.museum',
+ #'http://المغرب.icom.museum', // Morocco http://المغرب.icom.museum
+ 'http://नेपाल.icom.museum',
+ #'http://عمان.icom.museum', // Oman http://عمان.icom.museum
+ 'http://قطر.icom.museum',
+ 'http://românia.icom.museum',
+ 'http://россия.иком.museum',
+ 'http://србијаицрнагора.иком.museum',
+ 'http://இலங்கை.icom.museum',
+ 'http://españa.icom.museum',
+ 'http://ไทย.icom.museum',
+ 'http://تونس.icom.museum',
+ 'http://türkiye.icom.museum',
+ 'http://украина.icom.museum',
+ 'http://việtnam.icom.museum'
+ );
+
+ $blob ="
+ http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html
+
+ http://this.com
+
+ http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437
+
+ http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html
+
+ http://wordpress-core.com:8080/
+
+ http://www.website.com:5000
+
+ http://wordpress-core/?346236346326&2134362574863.437
+
+ http://افغانستا.icom.museum
+ http://الجزائر.icom.museum
+ http://österreich.icom.museum
+ http://বাংলাদেশ.icom.museum
+ http://беларусь.icom.museum
+ http://belgië.icom.museum
+ http://българия.icom.museum
+ http://تشادر.icom.museum
+ http://中国.icom.museum
+ http://českárepublika.icom.museum
+ http://ελλάδα.icom.museum
+ http://magyarország.icom.museum
+ http://ísland.icom.museum
+ http://भारत.icom.museum
+ http://ايران.icom.museum
+ http://éire.icom.museum
+ http://איקו״ם.ישראל.museum
+ http://日本.icom.museum
+ http://الأردن.icom.museum
+ http://қазақстан.icom.museum
+ http://한국.icom.museum
+ http://кыргызстан.icom.museum
+ http://ລາວ.icom.museum
+ http://لبنان.icom.museum
+ http://македонија.icom.museum
+ http://méxico.icom.museum
+ http://монголулс.icom.museum
+ http://नेपाल.icom.museum
+ http://قطر.icom.museum
+ http://românia.icom.museum
+ http://россия.иком.museum
+ http://србијаицрнагора.иком.museum
+ http://இலங்கை.icom.museum
+ http://españa.icom.museum
+ http://ไทย.icom.museum
+ http://تونس.icom.museum
+ http://türkiye.icom.museum
+ http://украина.icom.museum
+ http://việtnam.icom.museum
+ ";
+
+ $urls = wp_extract_urls( $blob );
+ $this->assertNotEmpty( $urls );
+ $this->assertInternalType( 'array', $urls );
+ $this->assertCount( count( $original_urls ), $urls );
+ $this->assertEquals( $original_urls, $urls );
+
+ $exploded = array_values( array_filter( array_map( 'trim', explode( "\n", $blob ) ) ) );
+ // wp_extract_urls calls html_entity_decode
+ $decoded = array_map( 'html_entity_decode', $exploded );
+
+ $this->assertEquals( $decoded, $urls );
+ $this->assertEquals( $original_urls, $decoded );
+
+ $blob ="Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+ incididunt ut labore http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html et dolore magna aliqua.
+ Ut http://this.com enim ad minim veniam, quis nostrud exercitation ullamco
+ laboris nisi ut aliquip ex http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437 ea
+ commodo consequat. http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html Duis aute irure dolor in reprehenderit in voluptate
+ velit esse http://wordpress-core.com:8080/ cillum dolore eu fugiat nulla <A href=\"http://www.website.com:5000\">http://www.website.com:5000</B> pariatur. Excepteur sint occaecat cupidatat non proident,
+ sunt in culpa qui officia deserunt mollit http://wordpress-core/?346236346326&2134362574863.437 anim id est laborum.";
+
+ $urls = wp_extract_urls( $blob );
+ $this->assertNotEmpty( $urls );
+ $this->assertInternalType( 'array', $urls );
+ $this->assertCount( 7, $urls );
+ $this->assertEquals( array_slice( $original_urls, 0, 7 ), $urls );
+
+ $blob = 'Lorem ipsum dolor sit amet, consectetur adipisicing elit, sed do eiusmod tempor
+ incididunt ut labore <a href="http://woo.com/1,2,3,4,5,6/-1-2-3-4-/woo.html">343462^</a> et dolore magna aliqua.
+ Ut <a href="http://this.com">&3640i6p1yi499</a> enim ad minim veniam, quis nostrud exercitation ullamco
+ laboris nisi ut aliquip ex <a href="http://www111.urwyeoweytwutreyytqytwetowteuiiu.com/?346236346326&2134362574863.437">343462^</a> ea
+ commodo consequat. <a href="http://wordpress-core/1,2,3,4,5,6/-1-2-3-4-/woo.html">343462^</a> Duis aute irure dolor in reprehenderit in voluptate
+ velit esse <a href="http://wordpress-core.com:8080/">-3-4--321-64-4@#!$^$!@^@^</a> cillum dolore eu <A href="http://www.website.com:5000">http://www.website.com:5000</B> fugiat nulla pariatur. Excepteur sint occaecat cupidatat non proident,
+ sunt in culpa qui officia deserunt mollit <a href="http://wordpress-core/?346236346326&2134362574863.437">)(*&^%$</a> anim id est laborum.';
+
+ $urls = wp_extract_urls( $blob );
+ $this->assertNotEmpty( $urls );
+ $this->assertInternalType( 'array', $urls );
+ $this->assertCount( 7, $urls );
+ $this->assertEquals( array_slice( $original_urls, 0, 7 ), $urls );
+ }
</ins><span class="cx"> }
</span></span></pre>
</div>
</div>
</body>
</html>