Merge branch '3.0-speed-up-uploads' into master-speed-up-uploads

CHANGELOG.md

@@ -1,5 +1,15 @@
 # Changelog
 
+## 2.0.24 - 2020-02-22
+
+- X509: fix PHP 5.3 compatability issue
+- SSH2: arcfour128 / arcfour256 were being included twice
+- SSH2: make window resizing behave more consistently with PuTTY (#1421)
+- SSH2: sodium_compat doesn't support memzero (#1432)
+- SSH2: logging enhancements
+- SFTP: don't buffer up download requests (PuTTY doesn't) (#1425)
+- RSA: make PSS verification work for key length that aren't a power of 2 (#1423)
+
 ## 2.0.23 - 2019-09-16
 
 - SSH2: fix regression for connecting to servers with bad hostnames (#1405)

phpseclib/Net/SFTP.php

@@ -238,7 +238,7 @@ class SFTP extends SSH2
      * @var array
      * @access private
      */
-    private $sortOptions = [];
+    protected $sortOptions = [];
 
     /**
      * Canonicalization Flag
@@ -401,6 +401,9 @@ class SFTP extends SSH2
         if (!defined('NET_SFTP_QUEUE_SIZE')) {
             define('NET_SFTP_QUEUE_SIZE', 32);
         }
+        if (!defined('NET_SFTP_UPLOAD_QUEUE_SIZE')) {
+            define('NET_SFTP_UPLOAD_QUEUE_SIZE', 1024);
+        }
     }
 
     /**
@@ -1060,28 +1063,6 @@ class SFTP extends SSH2
         }
     }
 
-    /**
-     * Returns the file size, in bytes, or false, on failure
-     *
-     * Files larger than 4GB will show up as being exactly 4GB.
-     *
-     * @param string $filename
-     * @return mixed
-     * @access public
-     */
-    public function size($filename)
-    {
-        if (!($this->bitmap & SSH2::MASK_LOGIN)) {
-            return false;
-        }
-
-        $result = $this->stat($filename);
-        if ($result === false) {
-            return false;
-        }
-        return isset($result['size']) ? $result['size'] : -1;
-    }
-
     /**
      * Save files / directories to cache
      *
@@ -1590,6 +1571,13 @@ class SFTP extends SSH2
                 }
 
                 $i++;
+
+                if ($i >= NET_SFTP_QUEUE_SIZE) {
+                    if (!$this->read_put_responses($i)) {
+                        return false;
+                    }
+                    $i = 0;
+                }
             }
         }
 
@@ -1599,8 +1587,11 @@ class SFTP extends SSH2
 
         $i++;
 
-        if (!$this->read_put_responses($i)) {
+        if ($i >= NET_SFTP_QUEUE_SIZE) {
-            return false;
+            if (!$this->read_put_responses($i)) {
+                return false;
+            }
+            $i = 0;
         }
 
         return true;
@@ -1946,7 +1937,7 @@ class SFTP extends SSH2
         $sftp_packet_size = 4096; // PuTTY uses 4096
         // make the SFTP packet be exactly 4096 bytes by including the bytes in the NET_SFTP_WRITE packets "header"
         $sftp_packet_size-= strlen($handle) + 25;
-        $i = 0;
+        $i = $j = 0;
         while ($dataCallback || ($size === 0 || $sent < $size)) {
             if ($dataCallback) {
                 $temp = call_user_func($dataCallback, $sftp_packet_size);
@@ -1962,7 +1953,7 @@ class SFTP extends SSH2
 
             $subtemp = $offset + $sent;
             $packet = pack('Na*N3a*', strlen($handle), $handle, $subtemp / 4294967296, $subtemp, strlen($temp), $temp);
-            if (!$this->send_sftp_packet(NET_SFTP_WRITE, $packet, $i)) {
+            if (!$this->send_sftp_packet(NET_SFTP_WRITE, $packet, $j)) {
                 if ($mode & self::SOURCE_LOCAL_FILE) {
                     fclose($fp);
                 }
@@ -1974,6 +1965,14 @@ class SFTP extends SSH2
             }
 
             $i++;
+            $j++;
+            if ($i == NET_SFTP_UPLOAD_QUEUE_SIZE) {
+                if (!$this->read_put_responses($i)) {
+                    $i = 0;
+                    break;
+                }
+                $i = 0;
+            }
         }
 
         if (!$this->read_put_responses($i)) {
@@ -2258,7 +2257,10 @@ class SFTP extends SSH2
                 return false;
             }
 
-            return $this->delete_recursive($path);
+            $i = 0;
+            $result = $this->delete_recursive($path, $i);
+            $this->read_put_responses($i);
+            return $result;
         }
 
         $this->remove_from_stat_cache($path);
@@ -2276,8 +2278,11 @@ class SFTP extends SSH2
      * @return bool
      * @access private
      */
-    private function delete_recursive($path)
+    private function delete_recursive($path, &$i)
     {
+        if (!$this->read_put_responses($i)) {
+            return false;
+        }
         $i = 0;
         $entries = $this->readlist($path, true);
 
@@ -2305,6 +2310,13 @@ class SFTP extends SSH2
                 $this->remove_from_stat_cache($temp);
 
                 $i++;
+
+                if ($i >= NET_SFTP_QUEUE_SIZE) {
+                    if (!$this->read_put_responses($i)) {
+                        return false;
+                    }
+                    $i = 0;
+                }
             }
         }
 
@@ -2315,8 +2327,11 @@ class SFTP extends SSH2
 
         $i++;
 
-        if (!$this->read_put_responses($i)) {
+        if ($i >= NET_SFTP_QUEUE_SIZE) {
-            return false;
+            if (!$this->read_put_responses($i)) {
+                return false;
+            }
+            $i = 0;
         }
 
         return true;

phpseclib/Net/SFTP/Stream.php

@@ -17,7 +17,7 @@
 
 namespace phpseclib3\Net\SFTP;
 
-use phpseclib3\Crypt\RSA;
+use phpseclib3\Crypt\Common\PrivateKey;
 use phpseclib3\Net\SFTP;
 use phpseclib3\Net\SSH2;
 
@@ -204,7 +204,7 @@ class Stream
             if (isset($context[$scheme]['password'])) {
                 $pass = $context[$scheme]['password'];
             }
-            if (isset($context[$scheme]['privkey']) && $context[$scheme]['privkey'] instanceof RSA) {
+            if (isset($context[$scheme]['privkey']) && $context[$scheme]['privkey'] instanceof PrivateKey) {
                 $pass = $context[$scheme]['privkey'];
             }
 
@@ -266,7 +266,7 @@ class Stream
         }
         $this->path = $path;
 
-        $this->size = $this->sftp->size($path);
+        $this->size = $this->sftp->filesize($path);
         $this->mode = preg_replace('#[bt]$#', '', $mode);
         $this->eof = false;
 

phpseclib/Net/SSH2.php

@@ -1295,6 +1295,7 @@ class SSH2
         }
 
         if (version_compare($matches[3], '1.99', '<')) {
+            $this->bitmap = 0;
             throw new UnableToConnectException("Cannot connect to SSH $matches[3] servers");
         }
 
@@ -1310,6 +1311,7 @@ class SSH2
             }
 
             if (!strlen($response) || ord($response[0]) != NET_SSH2_MSG_KEXINIT) {
+                $this->bitmap = 0;
                 throw new \UnexpectedValueException('Expected SSH_MSG_KEXINIT');
             }
 
@@ -1444,11 +1446,12 @@ class SSH2
 
             $kexinit_payload_server = $this->get_binary_packet();
             if ($kexinit_payload_server === false) {
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                 throw new ConnectionClosedException('Connection closed by server');
             }
 
             if (!strlen($kexinit_payload_server) || ord($kexinit_payload_server[0]) != NET_SSH2_MSG_KEXINIT) {
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_PROTOCOL_ERROR);
                 throw new \UnexpectedValueException('Expected SSH_MSG_KEXINIT');
             }
         }
@@ -1552,12 +1555,13 @@ class SSH2
 
                 $response = $this->get_binary_packet();
                 if ($response === false) {
-                    $this->bitmap = 0;
+                    $this->disconnect_helper(NET_SSH2_DISCONNECT_KEY_EXCHANGE_FAILED);
                     throw new ConnectionClosedException('Connection closed by server');
                 }
 
                 list($type, $primeBytes, $gBytes) = Strings::unpackSSH2('Css', $response);
                 if ($type != NET_SSH2_MSG_KEXDH_GEX_GROUP) {
+                    $this->disconnect_helper(NET_SSH2_DISCONNECT_PROTOCOL_ERROR);
                     throw new \UnexpectedValueException('Expected SSH_MSG_KEX_DH_GEX_GROUP');
                 }
                 $this->updateLogHistory('NET_SSH2_MSG_KEXDH_REPLY', 'NET_SSH2_MSG_KEXDH_GEX_GROUP');
@@ -1600,7 +1604,7 @@ class SSH2
 
         $response = $this->get_binary_packet();
         if ($response === false) {
-            $this->bitmap = 0;
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
             throw new ConnectionClosedException('Connection closed by server');
         }
         if (!strlen($response)) {
@@ -1615,6 +1619,7 @@ class SSH2
         ) = Strings::unpackSSH2('Csss', $response);
 
         if ($type != constant($serverKexReplyMessage)) {
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_PROTOCOL_ERROR);
             throw new \UnexpectedValueException("Expected $serverKexReplyMessage");
         }
         switch ($serverKexReplyMessage) {
@@ -1680,7 +1685,7 @@ class SSH2
                 case $this->signature_format == $server_host_key_algorithm:
                 case $server_host_key_algorithm != 'rsa-sha2-256' && $server_host_key_algorithm != 'rsa-sha2-512':
                 case $this->signature_format != 'ssh-rsa':
-                    $this->disconnect_helper(NET_SSH2_DISCONNECT_KEY_EXCHANGE_FAILED);
+                    $this->disconnect_helper(NET_SSH2_DISCONNECT_HOST_KEY_NOT_VERIFIABLE);
                     throw new \RuntimeException('Server Host Key Algorithm Mismatch');
             }
         }
@@ -1691,12 +1696,13 @@ class SSH2
         $response = $this->get_binary_packet();
 
         if ($response === false) {
-            $this->bitmap = 0;
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
             throw new ConnectionClosedException('Connection closed by server');
         }
 
         list($type) = Strings::unpackSSH2('C', $response);
         if ($type != NET_SSH2_MSG_NEWKEYS) {
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_PROTOCOL_ERROR);
             throw new \UnexpectedValueException('Expected SSH_MSG_NEWKEYS');
         }
 
@@ -2103,12 +2109,13 @@ class SSH2
                     }
                     return $this->login_helper($username, $password);
                 }
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                 throw new ConnectionClosedException('Connection closed by server');
             }
 
             list($type, $service) = Strings::unpackSSH2('Cs', $response);
             if ($type != NET_SSH2_MSG_SERVICE_ACCEPT || $service != 'ssh-userauth') {
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_PROTOCOL_ERROR);
                 throw new \UnexpectedValueException('Expected SSH_MSG_SERVICE_ACCEPT');
             }
             $this->bitmap |= self::MASK_LOGIN_REQ;
@@ -2147,7 +2154,7 @@ class SSH2
 
             $response = $this->get_binary_packet();
             if ($response === false) {
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                 throw new ConnectionClosedException('Connection closed by server');
             }
 
@@ -2195,7 +2202,7 @@ class SSH2
 
         $response = $this->get_binary_packet();
         if ($response === false) {
-            $this->bitmap = 0;
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
             throw new ConnectionClosedException('Connection closed by server');
         }
 
@@ -2269,7 +2276,7 @@ class SSH2
         } else {
             $orig = $response = $this->get_binary_packet();
             if ($response === false) {
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                 throw new ConnectionClosedException('Connection closed by server');
             }
         }
@@ -2391,7 +2398,12 @@ class SSH2
 
         if ($publickey instanceof RSA) {
             $privatekey = $privatekey->withPadding(RSA::SIGNATURE_PKCS1);
-            switch ($this->signature_format) {
+            $algos = ['rsa-sha2-256', 'rsa-sha2-512', 'ssh-rsa'];
+            if (isset($this->preferred['hostkey'])) {
+                $algos = array_intersect($this->preferred['hostkey'] , $algos);
+            }
+            $algo = self::array_intersect_first($algos, $this->server_host_key_algorithms);
+            switch ($algo) {
                 case 'rsa-sha2-512':
                     $hash = 'sha512';
                     $signatureType = 'rsa-sha2-512';
@@ -2455,7 +2467,7 @@ class SSH2
 
         $response = $this->get_binary_packet();
         if ($response === false) {
-            $this->bitmap = 0;
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
             throw new ConnectionClosedException('Connection closed by server');
         }
 
@@ -2483,7 +2495,7 @@ class SSH2
 
         $response = $this->get_binary_packet();
         if ($response === false) {
-            $this->bitmap = 0;
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
             throw new ConnectionClosedException('Connection closed by server');
         }
 
@@ -2596,7 +2608,7 @@ class SSH2
 
             $response = $this->get_binary_packet();
             if ($response === false) {
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                 throw new ConnectionClosedException('Connection closed by server');
             }
 
@@ -2721,7 +2733,7 @@ class SSH2
 
         $response = $this->get_binary_packet();
         if ($response === false) {
-            $this->bitmap = 0;
+            $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
             throw new ConnectionClosedException('Connection closed by server');
         }
 
@@ -2956,7 +2968,6 @@ class SSH2
         $this->channel_status[self::CHANNEL_SUBSYSTEM] = NET_SSH2_MSG_CHANNEL_REQUEST;
 
         $response = $this->get_channel_packet(self::CHANNEL_SUBSYSTEM);
-
         if ($response === false) {
             return false;
         }
@@ -3254,7 +3265,7 @@ class SSH2
         if ($this->hmac_check instanceof Hash) {
             $hmac = stream_get_contents($this->fsock, $this->hmac_size);
             if ($hmac === false || strlen($hmac) != $this->hmac_size) {
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_MAC_ERROR);
                 throw new \RuntimeException('Error reading socket');
             }
 
@@ -3264,10 +3275,12 @@ class SSH2
             if (($this->hmac_check->getHash() & "\xFF\xFF\xFF\xFF") == 'umac') {
                 $this->hmac_check->setNonce("\0\0\0\0" . pack('N', $this->get_seq_no));
                 if ($hmac != $this->hmac_check->hash($reconstructed)) {
+                    $this->disconnect_helper(NET_SSH2_DISCONNECT_MAC_ERROR);
                     throw new \RuntimeException('Invalid UMAC');
                 }
             } else {
                 if ($hmac != $this->hmac_check->hash(pack('Na*', $this->get_seq_no, $reconstructed))) {
+                    $this->disconnect_helper(NET_SSH2_DISCONNECT_MAC_ERROR);
                     throw new \RuntimeException('Invalid HMAC');
                 }
             }
@@ -3338,7 +3351,7 @@ class SSH2
         while ($remaining_length > 0) {
             $temp = stream_get_contents($this->fsock, $remaining_length);
             if ($temp === false || feof($this->fsock)) {
-                $this->bitmap = 0;
+                $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                 throw new \RuntimeException('Error reading from socket');
             }
             $buffer.= $temp;
@@ -3615,7 +3628,7 @@ class SSH2
 
                 $response = $this->get_binary_packet(true);
                 if ($response === false) {
-                    $this->bitmap = 0;
+                    $this->disconnect_helper(NET_SSH2_DISCONNECT_CONNECTION_LOST);
                     throw new ConnectionClosedException('Connection closed by server');
                 }
             }
@@ -4086,7 +4099,10 @@ class SSH2
     {
         if ($this->bitmap & self::MASK_CONNECTED) {
             $data = Strings::packSSH2('CNss', NET_SSH2_MSG_DISCONNECT, $reason, '', '');
-            $this->send_binary_packet($data);
+            try {
+                $this->send_binary_packet($data);
+            } catch (\Exception $e) {
+            }
         }
 
         $this->bitmap = 0;

tests/Functional/Net/SFTPLargeFileTest.php

@@ -36,7 +36,7 @@ class Functional_Net_SFTPLargeFileTest extends Functional_Net_SFTPTestCase
 
         $this->assertSame(
             128 * 1024 * 1024,
-            $this->sftp->size($filename),
+            $this->sftp->filesize($filename),
             'Failed asserting that uploaded local file has the expected length.'
         );
     }

tests/Functional/Net/SFTPStreamTest.php

@@ -24,7 +24,7 @@ class Functional_Net_SFTPStreamTest extends Functional_Net_SFTPTestCase
         $fp = fopen($this->buildUrl('fooo.txt'), 'wb', false, $context);
         $this->assertInternalType('resource', $fp);
         fclose($fp);
-        $this->assertSame(0, $this->sftp->size('fooo.txt'));
+        $this->assertSame(0, $this->sftp->filesize('fooo.txt'));
     }
 
     /**

tests/Functional/Net/SFTPUserStoryTest.php

@@ -148,7 +148,7 @@ class Functional_Net_SFTPUserStoryTest extends PhpseclibFunctionalTestCase
 
         $this->assertSame(
             self::$exampleDataLength,
-            $sftp->size('file1.txt'),
+            $sftp->filesize('file1.txt'),
             'Failed asserting that put example data has the expected length'
         );
 
@@ -184,7 +184,7 @@ class Functional_Net_SFTPUserStoryTest extends PhpseclibFunctionalTestCase
 
         $this->assertSame(
             self::$exampleDataLength,
-            $sftp->size('file1.txt'),
+            $sftp->filesize('file1.txt'),
             'Failed asserting that put example data has the expected length'
         );
 
@@ -232,7 +232,7 @@ class Functional_Net_SFTPUserStoryTest extends PhpseclibFunctionalTestCase
 
         $this->assertSame(
             1024 * 1024,
-            $sftp->size('file3.txt'),
+            $sftp->filesize('file3.txt'),
             'Failed asserting that truncate()\'d file has the expected length'
         );
 
@@ -352,7 +352,7 @@ class Functional_Net_SFTPUserStoryTest extends PhpseclibFunctionalTestCase
         $last_size = 0x7FFFFFFF;
         foreach ($files as $file) {
             if ($sftp->is_file($file)) {
-                $cur_size = $sftp->size($file);
+                $cur_size = $sftp->filesize($file);
                 $this->assertLessThanOrEqual(
                     $last_size,
                     $cur_size,
@@ -547,7 +547,7 @@ class Functional_Net_SFTPUserStoryTest extends PhpseclibFunctionalTestCase
         $filename = 'file-large-from-truncate-4112MiB.txt';
         $this->assertTrue($sftp->touch($filename));
         $this->assertTrue($sftp->truncate($filename, $filesize));
-        $this->assertSame($filesize, $sftp->size($filename));
+        $this->assertSame($filesize, $sftp->filesize($filename));
 
         return $sftp;
     }