#931 Fix: Legacy authentication breaks non-ASCII characters in passwords

Backport of #858

Author: mrotteveel

src/docs/asciidoc/release_notes.adoc

@@ -35,6 +35,20 @@ For known issues, consult <<known-issues>>.
 Future maintenance releases of Jaybird 5 may stop expanding Java support to include newly released Java versions.
 If you use Java 17 or higher, we recommend upgrading to https://firebirdsql.org/en/jdbc-driver/#jaybird6[Jaybird 6].
 
+[#jaybird-5-0-13-changelog]
+=== Jaybird 5.0.13
+
+The following has been changed or fixed since Jaybird 5.0.12:
+
+* Fixed: Legacy authentication fails with non-ASCII characters in password (https://github.com/FirebirdSQL/jaybird/issues/931[#931])
++
+This fix was backported from Jaybird 6.0.6.
++
+The hash that is sent instead of the actual password -- for authentication plugin `Legacy_Auth` or on Firebird 2.5 or older -- was incorrectly generated based on the `char` (UTF-16) value.
+We now use UTF-8 bytes instead.
++
+See also <<compat-legacy-auth>>.
+
 [#jaybird-5-0-12-changelog]
 === Jaybird 5.0.12
 
@@ -1859,3 +1873,15 @@ For more information about this library, see https://github.com/mrotteveel/jaybi
 
 In Jaybird 6 we will move the Type 2 support to a separate library.
 In the future we may provide JARs with the embedded libraries of a specific Firebird version.
+
+[#compat-legacy-auth]
+=== Legacy authentication in Jaybird 6.0.6 and higher
+
+Since Jaybird 5.0.13 (and Jaybird 6.0.6), legacy authentication -- Firebird 2.5 or older, or using authentication plugin `Legacy_Auth` -- uses UTF-8 to derive the bytes for calculating the hash of the password to send to the server.
+It is possible UTF-8 is not the right character set on Firebird 2.5 and older.
+This possibly also applies for Firebird 3.0 and higher, if the password was set using connection encoding NONE on (Windows) systems where windows-1252 or similar is the default encoding.
+
+You can use connection property `legacyAuthCharset` to specify a different Java character set name (for pure Java connections only).
+
+If authentication against Firebird 2.5 or older starts to fail for you, then likely you need to specify `legacyAuthCharset=iso-8859-1`.
+Resetting your password, using https://firebirdsql.org/file/documentation/chunk/en/refdocs/fblangref25/fblangref25-security.html#fblangref25-security-auth-alter-user[`ALTER USER`] with connection encoding UTF8, might also resolve such failures.

src/main/org/firebirdsql/ds/AbstractConnectionPropertiesDataSource.java

@@ -448,6 +448,16 @@ public void setParallelWorkers(int parallelWorkers) {
         FirebirdConnectionProperties.super.setParallelWorkers(parallelWorkers);
     }
 
+    @Override
+    public String getLegacyAuthCharset() {
+        return FirebirdConnectionProperties.super.getLegacyAuthCharset();
+    }
+
+    @Override
+    public void setLegacyAuthCharset(String legacyAuthCharset) {
+        FirebirdConnectionProperties.super.setLegacyAuthCharset(legacyAuthCharset);
+    }
+
     @Override
     public boolean isExtendedMetadata() {
         return FirebirdConnectionProperties.super.isExtendedMetadata();

src/main/org/firebirdsql/gds/ng/wire/auth/ClientAuthBlock.java

@@ -79,6 +79,10 @@ public String getPassword() {
         return attachProperties.getPassword();
     }
 
+    public String getLegacyAuthCharset() {
+        return attachProperties.getLegacyAuthCharset();
+    }
+
     public boolean isAuthComplete() {
         return authComplete;
     }

src/main/org/firebirdsql/gds/ng/wire/auth/legacy/LegacyAuthenticationPlugin.java

@@ -46,12 +46,15 @@ public String getName() {
         return LegacyAuthenticationPluginSpi.LEGACY_AUTH_NAME;
     }
 
+    @SuppressWarnings("deprecation")
     @Override
     public AuthStatus authenticate(ClientAuthBlock clientAuthBlock) throws SQLException {
         if (clientAuthBlock.getLogin() == null || clientAuthBlock.getPassword() == null) {
             return AuthStatus.AUTH_CONTINUE;
         }
-        clientData = UnixCrypt.crypt(clientAuthBlock.getPassword(), LEGACY_PASSWORD_SALT).substring(2, 13)
+        clientData = UnixCrypt
+                .crypt(clientAuthBlock.getPassword(), LEGACY_PASSWORD_SALT, clientAuthBlock.getLegacyAuthCharset())
+                .substring(2, 13)
                 .getBytes(StandardCharsets.US_ASCII);
         return AuthStatus.AUTH_SUCCESS;
     }

src/main/org/firebirdsql/gds/ng/wire/auth/legacy/UnixCrypt.java

@@ -25,6 +25,9 @@
 
 /* ------------------------------------------------------------ */
 
+import org.firebirdsql.logging.LoggerFactory;
+
+import java.nio.charset.Charset;
 import java.nio.charset.StandardCharsets;
 
 /**
@@ -34,6 +37,7 @@
  * @author Greg Wilkins (gregw)
  * @deprecated will be removed in Jaybird 6, and replaced by {@code LegacyHash} which will be introduced in Jaybird 6
  */
+@SuppressWarnings("DeprecatedIsStillUsed")
 @Deprecated
 public class UnixCrypt {
 
@@ -426,24 +430,31 @@ private static void init_perm(long[][] perm, byte[] p,int chars_out) {
     }
 
     /**
-     * Encrypts String into crypt (Unix) code.
-     * @param key the key to be encrypted
-     * @param setting the salt to be used
+     * Encrypts String into UnixCrypt code.
+     *
+     * @param key
+     *         the key to be encrypted
+     * @param setting
+     *         the salt to be used
+     * @param charsetName
+     *         character set to convert {@code key} to bytes
      * @return the encrypted String
+     * @since 5.0.13
      */
-    public static String crypt(String key, String setting)
-    {
+    public static String crypt(String key, String setting, String charsetName) {
+        if (key == null || setting == null) {
+            // will NOT match under ANY circumstances!
+            return "*";
+        }
         long constdatablock = 0L;		/* encryption constant */
         byte[] cryptresult = new byte[13];	/* encrypted result */
         long keyword = 0L;
-        /* invalid parameters! */
-        if(key==null||setting==null) 
-            return "*"; // will NOT match under ANY circumstances!
 
-        int keylen = key.length();
+        byte[] keyBytes = key.getBytes(toCharset(charsetName));
+        int keylen = keyBytes.length;
 
-        for (int i=0; i<8 ; i++) {
-            keyword = (keyword << 8) | ((i < keylen)? 2*key.charAt(i): 0);
+        for (int i = 0; i < 8; i++) {
+            keyword = (keyword << 8) | ((i < keylen) ? 2 * (keyBytes[i] & 0xFF) : 0);
         }
 
         long[] KS = des_setkey(keyword);
@@ -467,5 +478,15 @@ public static String crypt(String key, String setting)
         return new String(cryptresult, 0, 13, StandardCharsets.US_ASCII);
     }
 
+    private static Charset toCharset(String charsetName) {
+        try {
+            return Charset.forName(charsetName);
+        } catch (IllegalArgumentException e) {
+            LoggerFactory.getLogger(UnixCrypt.class)
+                    .warnfe("Invalid character set %s, falling back to UTF-8", charsetName, e);
+            return StandardCharsets.UTF_8;
+        }
+    }
+
 }
 

src/main/org/firebirdsql/gds/ng/wire/version10/V10ParameterConverter.java

@@ -39,6 +39,7 @@ public class V10ParameterConverter extends AbstractParameterConverter<WireDataba
 
     private static final String LEGACY_PASSWORD_SALT = "9z";
 
+    @SuppressWarnings("deprecation")
     @Override
     protected void populateAuthenticationProperties(final AbstractConnection<?, ?> connection,
             final ConnectionParameterBuffer pb) throws SQLException {
@@ -48,8 +49,9 @@ protected void populateAuthenticationProperties(final AbstractConnection<?, ?> c
             pb.addArgument(tagMapping.getUserNameTag(), props.getUser());
         }
         if (props.getPassword() != null) {
-            pb.addArgument(tagMapping.getEncryptedPasswordTag(), UnixCrypt.crypt(props.getPassword(),
-                    LEGACY_PASSWORD_SALT).substring(2, 13));
+            pb.addArgument(tagMapping.getEncryptedPasswordTag(),
+                    UnixCrypt.crypt(props.getPassword(), LEGACY_PASSWORD_SALT, props.getLegacyAuthCharset())
+                            .substring(2, 13));
         }
     }
 

src/main/org/firebirdsql/jaybird/props/AttachmentProperties.java

@@ -27,6 +27,8 @@
 import org.firebirdsql.gds.ng.FbAttachment;
 import org.firebirdsql.gds.ng.WireCrypt;
 
+import java.nio.charset.Charset;
+
 import static org.firebirdsql.jaybird.props.PropertyConstants.DEFAULT_WIRE_COMPRESSION;
 
 /**
@@ -432,4 +434,40 @@ default void setParallelWorkers(int parallelWorkers) {
         setIntProperty(PropertyNames.parallelWorkers, parallelWorkers);
     }
 
+    /**
+     * The Java character set to use when processing a password for Legacy_Auth in the pure Java protocol.
+     * <p>
+     * In general, this should be {@code UTF-8} (the default). Using a different character set may be needed when
+     * connecting to Firebird 2.5 and older when using passwords with non-ASCII characters.
+     * </p>
+     *
+     * @return Java character set name (default {@code UTF-8})
+     * @since 5.0.13
+     */
+    default String getLegacyAuthCharset() {
+        return getProperty(PropertyNames.legacyAuthCharset, PropertyConstants.DEFAULT_LEGACY_AUTH_CHARSET);
+    }
+
+    /**
+     * Sets the Java character set to use when processing a password for Legacy_Auth in the pure Java protocol.
+     * <p>
+     * In general, this should be {@code UTF-8} (the default). Using a different character set may be needed when
+     * connecting to Firebird 2.5 and older when using passwords with non-ASCII characters.
+     * </p>
+     *
+     * @param legacyAuthCharset
+     *         Java character set name (use {@code null} to reset to the default, {@code UTF-8})
+     * @throws java.nio.charset.IllegalCharsetNameException
+     *         if {@code legacyAuthCharset} is an illegal character set name
+     * @throws java.nio.charset.UnsupportedCharsetException
+     *         if {@code legacyAuth} is not supported
+     * @since 5.0.13
+     */
+    default void setLegacyAuthCharset(String legacyAuthCharset) {
+        if (legacyAuthCharset != null) {
+            Charset.forName(legacyAuthCharset);
+        }
+        setProperty(PropertyNames.legacyAuthCharset, legacyAuthCharset);
+    }
+
 }

src/main/org/firebirdsql/jaybird/props/PropertyConstants.java

@@ -76,6 +76,7 @@ public final class PropertyConstants {
     public static final String SESSION_TIME_ZONE_SERVER = "server";
 
     public static final String DEFAULT_AUTH_PLUGINS = "Srp256,Srp";
+    static final String DEFAULT_LEGACY_AUTH_CHARSET = "UTF-8";
 
     private PropertyConstants() {
         // no instances

src/main/org/firebirdsql/jaybird/props/PropertyNames.java

@@ -60,6 +60,7 @@ public final class PropertyNames {
     public static final String authPlugins = "authPlugins";
     public static final String wireCompression = "wireCompression";
     public static final String parallelWorkers = "parallelWorkers";
+    public static final String legacyAuthCharset = "legacyAuthCharset";
 
     // database connection
     public static final String sqlDialect = "sqlDialect";

src/test/org/firebirdsql/common/extension/DatabaseUserExtension.java

@@ -90,7 +90,7 @@ public void createUser(String username, String password, String plugin) throws S
             createUserSql.append(" USING PLUGIN ").append(plugin);
         }
 
-        try (Connection connection = getConnectionViaDriverManager();
+        try (Connection connection = getConnectionViaDriverManager("lc_ctype", "UTF8");
              Statement statement = connection.createStatement()) {
             statement.execute(createUserSql.toString());
         }
@@ -103,7 +103,7 @@ public void afterEach(ExtensionContext context) {
         if (createdUsers.isEmpty()) {
             return;
         }
-        try (Connection connection = getConnectionViaDriverManager()) {
+        try (Connection connection = getConnectionViaDriverManager("lc_ctype", "UTF8")) {
             connection.setAutoCommit(false);
             try (Statement statement = connection.createStatement()) {
                 for (User user : createdUsers) {