diff --git a/core/pom.xml b/core/pom.xml index 2c1e859f445..2d2506b4aa3 100644 --- a/core/pom.xml +++ b/core/pom.xml @@ -69,6 +69,18 @@ org.eclipse.angus jakarta.mail + + org.openjdk.jmh + jmh-core + 1.37 + test + + + org.openjdk.jmh + jmh-generator-annprocess + 1.37 + test + org.reactivestreams reactive-streams @@ -79,4 +91,32 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + + + + default-testCompile + + full + + + org.openjdk.jmh + jmh-generator-annprocess + 1.37 + + + + + + + + + diff --git a/core/src/main/java/org/apache/james/core/Domain.java b/core/src/main/java/org/apache/james/core/Domain.java index 50ad81c4126..2538113f4fa 100644 --- a/core/src/main/java/org/apache/james/core/Domain.java +++ b/core/src/main/java/org/apache/james/core/Domain.java @@ -20,6 +20,7 @@ package org.apache.james.core; import java.io.Serializable; +import java.net.IDN; import java.util.Locale; import java.util.Objects; @@ -42,6 +43,22 @@ public boolean matches(char c) { public static final Domain LOCALHOST = Domain.of("localhost"); public static final int MAXIMUM_DOMAIN_LENGTH = 253; + /** + * Whether {@code s} is pure US-ASCII. Defined here rather than in + * {@link MailAddress} -- which exposes it as public API -- because + * {@code Domain} must not depend on {@code MailAddress}: the two would + * then initialise each other, and {@link #LOCALHOST} is built in this + * class's static initialiser. + */ + static boolean isAscii(String s) { + for (int i = 0; i < s.length(); i++) { + if (s.charAt(i) > 0x7F) { + return false; + } + } + return true; + } + private static String removeBrackets(String domainName) { if (!(domainName.startsWith("[") && domainName.endsWith("]"))) { return domainName; @@ -54,9 +71,37 @@ public static Domain of(String domain) { Preconditions.checkArgument(domain.length() <= MAXIMUM_DOMAIN_LENGTH, "Domain name length should not exceed %s characters", MAXIMUM_DOMAIN_LENGTH); - String domainWithoutBrackets = removeBrackets(domain); + String withoutBrackets = removeBrackets(domain); + String domainWithoutBrackets; + if (isAscii(withoutBrackets)) { + // RFC 3490 4.1: ToASCII skips both nameprep and Punycode when the + // input is already all-ASCII and hands it back unchanged, so IDN + // would only cost time here -- and it is on the path of every + // address James parses. The empty-label and length checks it also + // performs are done below by assertValidPart anyway. + domainWithoutBrackets = withoutBrackets; + } else { + try { + domainWithoutBrackets = IDN.toASCII(withoutBrackets, IDN.ALLOW_UNASSIGNED); + } catch (IllegalArgumentException e) { + // IDN.toASCII's own message can be cryptic ("Empty label is + // not a legal name", "A prohibited code point was found in + // the input..."). Let's save wear and tear on the poor + // developer's brain. + throw new IllegalArgumentException( + "Domain '" + domain + "' is invalid according to IDNA: " + e.getMessage(), e); + } + } Preconditions.checkArgument(PART_CHAR_MATCHER.matchesAllOf(domainWithoutBrackets), - "Domain parts ASCII chars must be a-z A-Z 0-9 - or _ in %s", domain); + "Domain parts ASCII chars must be a-z A-Z 0-9 - or _ in %s", domain); + + if (domainWithoutBrackets.startsWith("xn--") || + domainWithoutBrackets.contains(".xn--")) { + domainWithoutBrackets = IDN.toUnicode(domainWithoutBrackets); + Preconditions.checkArgument(!domainWithoutBrackets.startsWith("xn--") && + !domainWithoutBrackets.contains(".xn--"), + "A-label could not be decoded to Unicode in %s", domain); + } int pos = 0; int nextDot = domainWithoutBrackets.indexOf('.'); diff --git a/core/src/main/java/org/apache/james/core/MailAddress.java b/core/src/main/java/org/apache/james/core/MailAddress.java index 41e88b2b90c..824a83cbfa1 100644 --- a/core/src/main/java/org/apache/james/core/MailAddress.java +++ b/core/src/main/java/org/apache/james/core/MailAddress.java @@ -19,6 +19,8 @@ package org.apache.james.core; +import java.net.IDN; +import java.text.Normalizer; import java.util.Locale; import java.util.Objects; import java.util.Optional; @@ -172,7 +174,16 @@ private int stripSourceRoute(String address, int pos) { * @throws AddressException if the parse failed */ public MailAddress(String address) throws AddressException { + // RFC 6532 §3.1 recommends NFC normalisation. Canonically-equivalent + // Unicode strings (for example U+00E9 vs U+0065 U+0301 — both render + // as "é") then produce equal MailAddress objects with equal hashCode + // values, which dedup, alias resolution and routing-table lookups + // rely on. NFC is a no-op for pure ASCII, and this runs for every + // address James parses, so ASCII skips the normaliser outright. address = address.trim(); + if (!Domain.isAscii(address)) { + address = Normalizer.normalize(address, Normalizer.Form.NFC); + } int pos = 0; // Test if mail address has source routing information (RFC-821) and get rid of it!! @@ -397,6 +408,24 @@ public String asString() { return localPart + "@" + domain.asString(); } + /** + * Whether this address is pure US-ASCII, i.e. it can be carried without + * RFC 6531 (SMTPUTF8) and reported with the RFC 3798 {@code rfc822} + * addr-type rather than the RFC 6533 {@code utf-8} one. + */ + public boolean isAscii() { + return isAscii(localPart) && isAscii(domain.asString()); + } + + /** + * Whether {@code s} is pure US-ASCII. Shared by the callers that hold an + * address as a raw string rather than as a {@link MailAddress} -- the SMTP + * command handlers and the remote-delivery SMTPUTF8 strategy. + */ + public static boolean isAscii(String s) { + return Domain.isAscii(s); + } + @Override public String toString() { return localPart + "@" + Optional.ofNullable(domain) @@ -418,7 +447,7 @@ public Optional toInternetAddress() { try { return Optional.of(new InternetAddress(toString())); } catch (AddressException ae) { - LOGGER.warn("A valid address '{}' as per James criterial fails to parse as a jakarta.mail InternetAdrress", asString()); + LOGGER.warn("A valid address '{}' as per James criteria fails to parse as a jakarta.mail InternetAdrress", asString()); return Optional.empty(); } } @@ -513,6 +542,24 @@ private int parseQuotedLocalPart(StringBuilder lpSB, String address, int pos) th "characters exception , , quote (\"), or backslash (\\) at position " + (pos + 1) + " in '" + address + "'"); } + // Same surrogate-pair check as in parseUnquotedLocalPart: + // unpaired or mis-ordered surrogates would produce + // ill-formed UTF-8 on output. + if (Character.isLowSurrogate(q)) { + throw new AddressException("Unpaired UTF-16 low surrogate in quoted local-part at position " + + (pos + 1) + " in '" + address + "'"); + } + if (Character.isHighSurrogate(q)) { + if (pos + 1 >= address.length() + || !Character.isLowSurrogate(address.charAt(pos + 1))) { + throw new AddressException("Unpaired UTF-16 high surrogate in quoted local-part at position " + + (pos + 1) + " in '" + address + "'"); + } + lpSB.append(q); + lpSB.append(address.charAt(pos + 1)); + pos += 2; + continue; + } lpSB.append(q); pos++; } @@ -549,18 +596,40 @@ private int parseUnquotedLocalPart(StringBuilder lpSB, String address, int pos) //End of local-part break; } else { - // ::= any one of the 128 ASCII characters, but not any - // or + // ::= any printable ASCII character, or any non-ASCII + // unicode codepoint, but not or // ::= "<" | ">" | "(" | ")" | "[" | "]" | "\" | "." // | "," | ";" | ":" | "@" """ | the control - // characters (ASCII codes 0 through 31 inclusive and - // 127) + // characters (ASCII codes 0 through 31 inclusive, + // 127, and the C1 controls 128 through 159) // ::= the space character (ASCII code 32) char c = address.charAt(pos); - if (c <= 31 || c >= 127 || c == ' ') { + if (c <= 31 || c == 127 || c == ' ' || (c >= 0x80 && c <= 0x9F)) { throw new AddressException("Invalid character in local-part (user account) at position " + (pos + 1) + " in '" + address + "'", address, pos + 1); } + // Java strings are UTF-16, so a supplementary-plane + // codepoint (emoji, CJK extension, etc.) appears here as a + // high-surrogate followed by a low-surrogate. We must keep + // them paired so the address can serialise as well-formed + // UTF-8 (RFC 6532 §3.1) — a lone or mis-ordered surrogate + // would produce ill-formed UTF-8 octets on output. + if (Character.isLowSurrogate(c)) { + throw new AddressException("Unpaired UTF-16 low surrogate in local-part at position " + + (pos + 1) + " in '" + address + "'", address, pos + 1); + } + if (Character.isHighSurrogate(c)) { + if (pos + 1 >= address.length() + || !Character.isLowSurrogate(address.charAt(pos + 1))) { + throw new AddressException("Unpaired UTF-16 high surrogate in local-part at position " + + (pos + 1) + " in '" + address + "'", address, pos + 1); + } + lpSB.append(c); + lpSB.append(address.charAt(pos + 1)); + pos += 2; + lastCharDot = false; + continue; + } int i = 0; while (i < SPECIAL.length) { if (c == SPECIAL[i]) { @@ -688,6 +757,7 @@ private int parseDomain(StringBuilder dSB, String address, int pos) throws Addre // in practice though, we should relax this as domain names can start // with digits as well as letters. So only check that doesn't start // or end with hyphen. + boolean unicode = false; while (true) { if (pos >= address.length()) { break; @@ -700,6 +770,11 @@ private int parseDomain(StringBuilder dSB, String address, int pos) throws Addre resultSB.append(ch); pos++; continue; + } else if (ch >= 0x0080) { + resultSB.append(ch); + pos++; + unicode = true; + continue; } if (ch == '.') { break; @@ -707,6 +782,19 @@ private int parseDomain(StringBuilder dSB, String address, int pos) throws Addre throw new AddressException("Invalid character at " + pos + " in '" + address + "'", address, pos); } String result = resultSB.toString(); + if (unicode) { + try { + result = IDN.toASCII(result, IDN.ALLOW_UNASSIGNED); + } catch (IllegalArgumentException e) { + throw new AddressException("Domain invalid according to IDNA", address); + } + } + if (result.startsWith("xn--") || result.contains(".xn--")) { + result = IDN.toUnicode(result); + if (result.startsWith("xn--") || result.contains(".xn--")) { + throw new AddressException("Domain invalid according to IDNA", address); + } + } if (result.startsWith("-") || result.endsWith("-")) { throw new AddressException("Domain name cannot begin or end with a hyphen \"-\" at position " + (pos + 1) + " in '" + address + "'", address, pos + 1); diff --git a/core/src/test/java/org/apache/james/core/DomainTest.java b/core/src/test/java/org/apache/james/core/DomainTest.java new file mode 100644 index 00000000000..a1aa69d5c2b --- /dev/null +++ b/core/src/test/java/org/apache/james/core/DomainTest.java @@ -0,0 +1,91 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.core; + +import static org.assertj.core.api.Assertions.assertThat; +import static org.assertj.core.api.Assertions.assertThatThrownBy; + +import java.util.stream.Stream; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; + + +class DomainTest { + @Test + void testPlainDomain() { + Domain d1 = Domain.of("example.com"); + assertThat(d1.name().equals(d1.asString())); + Domain d2 = Domain.of("Example.com"); + assertThat(d2.name()).isNotEqualTo(d2.asString()); + assertThat(d1.asString()).isEqualTo(d2.asString()); + } + + @Test + void testIPv4Domain() { + Domain d1 = Domain.of("192.0.4.1"); + assertThat(d1.asString()).isEqualTo("192.0.4.1"); + } + + @Test + void testPunycodeIDN() { + Domain d1 = Domain.of("xn--gr-zia.example"); + assertThat(d1.asString()).isEqualTo("grå.example"); + } + + @Test + void testDevanagariDomain() { + Domain d1 = Domain.of("डाटामेल.भारत"); + assertThat(d1.asString()).isEqualTo(d1.name()); + } + + private static Stream malformedDomains() { + return Stream.of( + "😊☺️.example", // emoji not permitted by IDNA + "#.example", // really and truly not permitted + "\uFEFF.example", // U+FEFF is the byte order mark + "\u200C.example", // U+200C is a zero-width non-joiner + "\u200Eibm.example" // U+200E is left-to-right + ) + .map(Arguments::of); + } + + @ParameterizedTest + @MethodSource("malformedDomains") + void testMalformedDomains(String malformed) { + assertThatThrownBy(() -> Domain.of(malformed)) + .as("rejecting malformed domain " + malformed) + .isInstanceOf(IllegalArgumentException.class); + } + + @ParameterizedTest + @MethodSource("malformedDomains") + void exceptionForMalformedDomainShouldNameTheOffendingInput(String malformed) { + // Without the offending input in the message, "Domain invalid + // according to IDNA" leaves a future debugger guessing what + // string actually triggered it. + assertThatThrownBy(() -> Domain.of(malformed)) + .hasMessageContaining(malformed); + } +} + + diff --git a/core/src/test/java/org/apache/james/core/MailAddressAsciiBenchmark.java b/core/src/test/java/org/apache/james/core/MailAddressAsciiBenchmark.java new file mode 100644 index 00000000000..c07bed933a5 --- /dev/null +++ b/core/src/test/java/org/apache/james/core/MailAddressAsciiBenchmark.java @@ -0,0 +1,95 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.core; + +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +/** + * The addresses James sees the most: pure US-ASCII. RFC 6532 support must not + * make this path more expensive, and it very nearly did -- {@code Domain.of} + * used to run {@link java.net.IDN#toASCII} over every domain, ASCII or not, + * which cost 5x on this benchmark. + * + * Deliberately uses no API beyond what an unpatched James exposes, so the same + * class can be run against another revision of james-core to compare. + * + * Not run by the build: remove {@link Disabled} to measure locally. + */ +public class MailAddressAsciiBenchmark { + @Test + @Disabled("JMH benchmark, run on demand rather than on every build") + public void launchBenchmark() throws Exception { + Options opt = new OptionsBuilder() + .include(this.getClass().getName() + ".*") + .mode(Mode.AverageTime) + .timeUnit(TimeUnit.NANOSECONDS) + .warmupTime(TimeValue.seconds(1)) + .warmupIterations(5) + .measurementTime(TimeValue.seconds(1)) + .measurementIterations(10) + .threads(1) + .forks(2) + .shouldFailOnError(true) + .shouldDoGC(true) + .build(); + + new Runner(opt).run(); + } + + @Benchmark + public void shortAsciiAddress(Blackhole bh) throws Exception { + bh.consume(new MailAddress("server-dev@james.apache.org")); + } + + @Benchmark + public void asciiAddressWithDetails(Blackhole bh) throws Exception { + bh.consume(new MailAddress("user+mailbox/department=shipping@subdomain.example.com")); + } + + @Benchmark + public void longAsciiAddress(Blackhole bh) throws Exception { + bh.consume(new MailAddress("a-fairly-long-local-part-as-mailing-lists-generate@lists.deeply.nested.subdomain.example.org")); + } + + @Benchmark + public void quotedAsciiLocalPart(Blackhole bh) throws Exception { + bh.consume(new MailAddress("\"Fred Bloggs\"@example.com")); + } + + @Benchmark + public void asciiDomain(Blackhole bh) { + bh.consume(Domain.of("lists.deeply.nested.subdomain.example.org")); + } + + @Benchmark + public void asciiAddressAsString(Blackhole bh) throws Exception { + bh.consume(new MailAddress("server-dev@james.apache.org").asString()); + } +} diff --git a/core/src/test/java/org/apache/james/core/MailAddressTest.java b/core/src/test/java/org/apache/james/core/MailAddressTest.java index 86e5adece4b..144161e5fa2 100644 --- a/core/src/test/java/org/apache/james/core/MailAddressTest.java +++ b/core/src/test/java/org/apache/james/core/MailAddressTest.java @@ -22,12 +22,15 @@ import static org.assertj.core.api.Assertions.assertThat; import static org.assertj.core.api.Assertions.assertThatCode; +import java.util.Properties; import java.util.stream.Stream; +import jakarta.mail.Session; import jakarta.mail.internet.AddressException; import jakarta.mail.internet.InternetAddress; import org.assertj.core.api.Assertions; +import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.Arguments; @@ -37,6 +40,13 @@ class MailAddressTest { + // Checkstyle forbids \\uXXXX escapes, and lone surrogates or a bare combining + // mark cannot be spelled out as source literals, so build them by code point. + private static final String HIGH_SURROGATE = String.valueOf(Character.highSurrogate(0x1F600)); + private static final String LOW_SURROGATE = String.valueOf(Character.lowSurrogate(0x1F600)); + private static final String COMBINING_ACUTE = String.valueOf((char) 0x0301); + private static final String COMBINING_RING_ABOVE = String.valueOf((char) 0x030A); + private static final String GOOD_LOCAL_PART = "\"quoted@local part\""; private static final String GOOD_QUOTED_LOCAL_PART = "\"quoted@local part\"@james.apache.org"; private static final String GOOD_ADDRESS = "server-dev@james.apache.org"; @@ -55,6 +65,16 @@ private static Stream goodAddresses() { "\\.server-dev@james.apache.org", "Abc@10.42.0.1", "Abc.123@example.com", + "Loïc.Accentué@voilà.fr8", + // Supplementary-plane codepoint as a properly-paired + // UTF-16 surrogate pair (U+1F600). + "abc" + HIGH_SURROGATE + LOW_SURROGATE + "@example.com", + "pelé@exemple.com", + "δοκιμή@παράδειγμα.δοκιμή", + "我買@屋企.香港", + "二ノ宮@黒川.日本", + "медведь@с-балалайкой.рф", + //"संपर्क@डाटामेल.भारत", fails in Jakarta, reason still unknown "user+mailbox/department=shipping@example.com", "user+mailbox@example.com", "\"Abc@def\"@example.com", @@ -96,26 +116,45 @@ private static Stream badAddresses() { "server-dev@[127.0.1.1.1]", "server-dev@[127.0.1.-1]", "test@dom+ain.com", + "test@xn--.example", "\"a..b\"@domain.com", // jakarta.mail is unable to handle this so we better reject it "server-dev\\.@james.apache.org", // jakarta.mail is unable to handle this so we better reject it "a..b@domain.com", - // According to wikipedia these addresses are valid but as jakarta.mail is unable - // to work with them we shall rather reject them (note that this is not breaking retro-compatibility) - "Loïc.Accentué@voilà.fr8", - "pelé@exemple.com", - "δοκιμή@παράδειγμα.δοκιμή", - "我買@屋企.香港", - "二ノ宮@黒川.日本", - "медведь@с-балалайкой.рф", - "संपर्क@डाटामेल.भारत", + "sales@\u200Eibm.example", // U+200E is left-to-right + // Unpaired and mis-ordered UTF-16 surrogates would + // produce ill-formed UTF-8 on output, so we reject + // them: lone high surrogate at end, lone low + // surrogate, and a high surrogate not followed by + // a low one. + "abc" + HIGH_SURROGATE + "@example.com", + "abc" + LOW_SURROGATE + "def@example.com", + "abc" + HIGH_SURROGATE + "def@example.com", + // C1 controls in the local part: rejected on the + // same grounds as C0. Tested with U+0080 (start), + // U+0085 (NEL — common in EBCDIC interop bugs), + // and U+009F (end of the C1 range). + "abc\u0080def@example.com", + "abc\u0085def@example.com", + "abc\u009Fdef@example.com", + // According to wikipedia this address is valid but as jakarta.mail is unable + // to work with it we shall rather reject them (note that this is not breaking retro-compatibility) "mail.allow\\,d@james.apache.org") .map(Arguments::of); } + @BeforeEach + void setup() { + Properties props = new Properties(); + props.setProperty("mail.mime.allowutf8", "true"); + Session s = Session.getDefaultInstance(props); + assertThat(Boolean.parseBoolean(s.getProperties().getProperty("mail.mime.allowutf8", "false"))); + } + @ParameterizedTest @MethodSource("goodAddresses") void testGoodMailAddressString(String mailAddress) { assertThatCode(() -> new MailAddress(mailAddress)) + .as("parses " + mailAddress) .doesNotThrowAnyException(); } @@ -123,6 +162,7 @@ void testGoodMailAddressString(String mailAddress) { @MethodSource("goodAddresses") void toInternetAddressShouldNoop(String mailAddress) throws Exception { assertThat(new MailAddress(mailAddress).toInternetAddress()) + .as("tries to parse " + mailAddress + " using jakarta.mail") .isNotEmpty(); } @@ -130,6 +170,7 @@ void toInternetAddressShouldNoop(String mailAddress) throws Exception { @MethodSource("badAddresses") void testBadMailAddressString(String mailAddress) { Assertions.assertThatThrownBy(() -> new MailAddress(mailAddress)) + .as("fails to parse " + mailAddress) .isInstanceOf(AddressException.class); } @@ -313,4 +354,58 @@ void stripDetailsShouldBePreciseWithMultipleCharacterDelimiter() throws AddressE assertThat(mailAddress.stripDetails("--")).isEqualTo("localpart@example.com"); } + // RFC 6532 §3.1 — NFC normalisation + + @Test + void nfcAndNfdFormsOfSameAddressShouldCompareEqual() throws AddressException { + // "pelé" as NFC: p, e, l, U+00E9 + MailAddress nfc = new MailAddress("pelé@example.com"); + // "pelé" as NFD: p, e, l, U+0065, U+0301 (combining acute) + MailAddress nfd = new MailAddress("pelé@example.com"); + + assertThat(nfc).isEqualTo(nfd); + assertThat(nfc.hashCode()).isEqualTo(nfd.hashCode()); + assertThat(nfc.asString()).isEqualTo(nfd.asString()); + } + + @Test + void nfdInputShouldBeStoredAsNfc() throws AddressException { + // Build the input string explicitly in NFD form: the local + // part is p, e, l, e, U+0301 (combining acute) — five codepoints. + String input = "pele" + COMBINING_ACUTE + "@example.com"; + int atIndex = input.indexOf('@'); + int lIndex = input.indexOf('l'); + + // Sanity-check that the input is actually NFD: there should be + // two codepoints between 'l' and '@' (the 'e' and the + // combining acute). + assertThat(input.codePointCount(lIndex + 1, atIndex)).isEqualTo(2); + + MailAddress address = new MailAddress(input); + + // The local part comes back in NFC form (single U+00E9), not + // the two codepoints that went in. + assertThat(address.getLocalPart()).isEqualTo("pelé"); + assertThat(address.getLocalPart().codePointAt(3)).isEqualTo(0x00E9); + } + + @Test + void nfcNormalisationShouldBeNoopForAsciiAddresses() throws AddressException { + MailAddress address = new MailAddress("arnt@example.com"); + + assertThat(address.asString()).isEqualTo("arnt@example.com"); + } + + @Test + void nfcNormalisationShouldApplyToDomainsToo() throws AddressException { + // Unicode combining sequence in the domain's Unicode form. After + // construction, asString() should round-trip through NFC (whether the + // domain is ultimately stored as A-label or U-label is the Domain + // class's concern — here we only check that the two spellings collapse). + MailAddress nfc = new MailAddress("info@grå.org"); + MailAddress nfd = new MailAddress("info@gra" + COMBINING_RING_ABOVE + ".org"); + + assertThat(nfc).isEqualTo(nfd); + } + } diff --git a/core/src/test/java/org/apache/james/core/MailAddressUnicodeBenchmark.java b/core/src/test/java/org/apache/james/core/MailAddressUnicodeBenchmark.java new file mode 100644 index 00000000000..242fcdce1cd --- /dev/null +++ b/core/src/test/java/org/apache/james/core/MailAddressUnicodeBenchmark.java @@ -0,0 +1,96 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.core; + +import java.util.concurrent.TimeUnit; + +import org.junit.jupiter.api.Disabled; +import org.junit.jupiter.api.Test; +import org.openjdk.jmh.annotations.Benchmark; +import org.openjdk.jmh.annotations.Mode; +import org.openjdk.jmh.infra.Blackhole; +import org.openjdk.jmh.runner.Runner; +import org.openjdk.jmh.runner.options.Options; +import org.openjdk.jmh.runner.options.OptionsBuilder; +import org.openjdk.jmh.runner.options.TimeValue; + +/** + * RFC 6532 addresses. There is no before/after to compare here -- James used + * to reject these outright -- only the absolute cost of the new path. + * + * Unicode domains are the expensive part by a wide margin: {@code Domain.of} + * runs {@link java.net.IDN#toASCII} and then, seeing the {@code xn--} labels + * it just produced, {@link java.net.IDN#toUnicode} to get back, so a Unicode + * domain pays for a full Punycode round trip. + * + * Not run by the build: remove {@link Disabled} to measure locally. + */ +public class MailAddressUnicodeBenchmark { + // Checkstyle forbids \\uXXXX escapes and a bare combining mark cannot be + // spelled out as a source literal, so build it by code point. + private static final String COMBINING_ACUTE = String.valueOf((char) 0x0301); + + @Test + @Disabled("JMH benchmark, run on demand rather than on every build") + public void launchBenchmark() throws Exception { + Options opt = new OptionsBuilder() + .include(this.getClass().getName() + ".*") + .mode(Mode.AverageTime) + .timeUnit(TimeUnit.NANOSECONDS) + .warmupTime(TimeValue.seconds(1)) + .warmupIterations(5) + .measurementTime(TimeValue.seconds(1)) + .measurementIterations(10) + .threads(1) + .forks(2) + .shouldFailOnError(true) + .shouldDoGC(true) + .build(); + + new Runner(opt).run(); + } + + /** Already NFC: normalisation should be skipped outright. */ + @Benchmark + public void unicodeAddressAlreadyNfc(Blackhole bh) throws Exception { + bh.consume(new MailAddress("pelé@exemple.com")); + } + + /** NFD input: the normaliser has real work to do. */ + @Benchmark + public void unicodeAddressNeedingNfc(Blackhole bh) throws Exception { + bh.consume(new MailAddress("pele" + COMBINING_ACUTE + "@example.com")); + } + + @Benchmark + public void cjkAddress(Blackhole bh) throws Exception { + bh.consume(new MailAddress("二ノ宮@黒川.日本")); + } + + @Benchmark + public void unicodeDomain(Blackhole bh) { + bh.consume(Domain.of("παράδειγμα.δοκιμή")); + } + + @Benchmark + public void aceDomain(Blackhole bh) { + bh.consume(Domain.of("xn--hxajbheg2az3al.xn--jxalpdlp")); + } +} diff --git a/mailbox/opensearch/src/test/resources/eml/cve-2024-23184.eml b/mailbox/opensearch/src/test/resources/eml/cve-2024-23184.eml new file mode 100644 index 00000000000..30b57dd7430 --- /dev/null +++ b/mailbox/opensearch/src/test/resources/eml/cve-2024-23184.eml @@ -0,0 +1,45 @@ +MIME-Version: 1.0 +Subject: Test +From: Benoit TELLIER +To: Benoit TELLIER +Date: Tue, 13 Feb 2024 23:01:18 +0000 +Message-ID: +Content-Type: multipart/mixed; + boundary="-=Part.17f.732e3d28e1c76db4.18da4b40791.62ef5e3fa995057d=-" + +---=Part.17f.732e3d28e1c76db4.18da4b40791.62ef5e3fa995057d=- +Content-Type: multipart/alternative; + boundary="-=Part.17e.48ac92d73c356567.18da4b40791.360a293e2f389efe=-" + +---=Part.17e.48ac92d73c356567.18da4b40791.360a293e2f389efe=- +Content-Type: text/plain; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +Test + +---=Part.17e.48ac92d73c356567.18da4b40791.360a293e2f389efe=- +Content-Type: text/html; charset=UTF-8 +Content-Transfer-Encoding: quoted-printable + +
Test

+ +---=Part.17e.48ac92d73c356567.18da4b40791.360a293e2f389efe=--- + +---=Part.17f.732e3d28e1c76db4.18da4b40791.62ef5e3fa995057d=- +Content-Type: application/json; name="=?US-ASCII?Q?id=5Frsa.txt?=" +Content-Disposition: attachment +Content-Transfer-Encoding: base64 + +c3NoLXJzYSBBQUFBQjNOemFDMXljMkVBQUFBREFRQUJBQUFDQVFDa0dXMkp5c2lKR2hQZXdBOXRr +bVFFQm5EVjRaQ0llLy92ZFoyV0RybnZiNlZLQzdpWldjODFpU1ZkTFcxUkRBTll4c3ExN0dQanpV +OFlWdk9sRkFJSk1WTm9ESWhuQWtYOU9VUUJpd1hpOHlHZ3FLNGR0RmIxczJBRzNrQmxNUFFJOE5K +MkpLT2Z5MW51VnJubEtoVDlCVnpYMm5iSjNOak9PZlkxQlJEaDZZcVl1a2RuejBUT2k1Rkp1YUJT +NDZQemx3eWdIa0dzeXBLVHM2Y2FUNjBRdjl3eWFadm4yenN1RmNML3o2Mmd3aGZyZGFsakF1UGRX +cERlNG1IRVFmMXA2SXNRMDdPb0lwTmRHQ0tLZHRZQlVTcktzTXRpMllLUGZpSzB2WGU1L3owRWJE +VlRja1BrY3NwQ2cwYVZuZTB2eFVsRGt2U2pwV2tiQkZ0YTk5ekJjOVlJL0ROK28vRmtONlFTdXV5 +U29tNDZkamZpUjdqSzNMRmJKUkhaem9BblNvaTZvRlR0MW1LWjNzam44bnZWUG1PV3pJWHY0Tm1O +R1ExZHFrV1hXcUtyQjlIZUZiQnRPWVAzaEkxQ0kvaVhNbVR1SkdvcHVTUmlTNW1QZXlSQWV6VGtk +UG8vZ2NSVWNzbklhVW1EallUWHBFNzU3Yk5LWVNHbFJsS3FrbEhKc2JveEdTK0NaVzBJS2dZeTdG +cmZRZ1FGMTdvaUpWM1JJQ1VHcU9rM1I2VnZOYlhlL2VmZS9IT24xd0lZUS9qVGRzY0hCamRIM2FF +MmY4Y3dVS1IzNUtWNlJ1SE4vYVpiekxiVkJxUEMvUTcwd3NMQlloV29Da1dRMElUUmxGV2N3bnN3 +VTE5NnlGWkVHSmthOUNEaHZQdUVBV0NLWnFRT3gyMnRoYWVSQlE9PSBiZW53YUBob3Jpem9uCg== \ No newline at end of file diff --git a/mdn/src/main/java/org/apache/james/mdn/MDN.java b/mdn/src/main/java/org/apache/james/mdn/MDN.java index a8842396283..2311a0638d4 100644 --- a/mdn/src/main/java/org/apache/james/mdn/MDN.java +++ b/mdn/src/main/java/org/apache/james/mdn/MDN.java @@ -38,6 +38,7 @@ import org.apache.commons.io.IOUtils; import org.apache.james.javax.MimeMultipartReport; +import org.apache.james.mdn.fields.AddressType; import org.apache.james.mime4j.Charsets; import org.apache.james.mime4j.dom.Entity; import org.apache.james.mime4j.dom.Message; @@ -59,9 +60,15 @@ public class MDN { private static final NameValuePair UTF_8_CHARSET = new NameValuePair("charset", Charsets.UTF_8.name()); public static final String DISPOSITION_CONTENT_TYPE = "message/disposition-notification"; + public static final String GLOBAL_DISPOSITION_CONTENT_TYPE = "message/global-disposition-notification"; public static final String REPORT_SUB_TYPE = "report"; public static final String DISPOSITION_NOTIFICATION_REPORT_TYPE = "disposition-notification"; + private static boolean isDispositionNotificationType(String mimeType) { + return mimeType.equals(DISPOSITION_CONTENT_TYPE) + || mimeType.equals(GLOBAL_DISPOSITION_CONTENT_TYPE); + } + public static class Builder { private String humanReadableText; private MDNReport report; @@ -170,11 +177,11 @@ public static Optional extractHumanReadableText(List entities) t public static Optional extractMDNReport(List entities) { return entities.stream() - .filter(entity -> entity.getMimeType().startsWith(DISPOSITION_CONTENT_TYPE)) + .filter(entity -> isDispositionNotificationType(entity.getMimeType())) .findAny() .flatMap(entity -> { try (InputStream inputStream = ((SingleBody) entity.getBody()).getInputStream()) { - Try result = MDNReportParser.parse(inputStream, entity.getCharset()); + Try result = MDNReportParser.parse(inputStream, reportCharset(entity)); if (result.isSuccess()) { return Optional.of(result.get()); } else { @@ -186,8 +193,21 @@ public static Optional extractMDNReport(List entities) { }); } + /** + * RFC 6533 defines {@code message/global-disposition-notification} as UTF-8; + * no charset parameter is defined for it, so senders legitimately omit one and + * mime4j then falls back to us-ascii. Only the RFC 3798 form honours the + * declared charset. + */ + private static String reportCharset(Entity entity) { + if (GLOBAL_DISPOSITION_CONTENT_TYPE.equals(entity.getMimeType())) { + return StandardCharsets.UTF_8.name(); + } + return entity.getCharset(); + } + public boolean isReport(Entity entity) { - return entity.getMimeType().startsWith(DISPOSITION_CONTENT_TYPE); + return isDispositionNotificationType(entity.getMimeType()); } private final String humanReadableText; @@ -245,10 +265,26 @@ public BodyPart computeHumanReadablePart() throws MessagingException { public BodyPart computeReportPart() throws MessagingException { MimeBodyPart mdnPart = new MimeBodyPart(); - mdnPart.setContent(report.formattedValue(), DISPOSITION_CONTENT_TYPE); + mdnPart.setContent(report.formattedValue(), dispositionContentType()); return mdnPart; } + /** + * Per RFC 6533, emits {@code message/global-disposition-notification} when + * any recipient in the report uses the {@code utf-8} addr-type, otherwise + * the RFC 3798 form {@code message/disposition-notification}. + */ + private String dispositionContentType() { + boolean finalIsUtf8 = report.getFinalRecipientField().getAddressType() + .equals(AddressType.UTF_8); + boolean originalIsUtf8 = report.getOriginalRecipientField() + .map(r -> r.getAddressType().equals(AddressType.UTF_8)) + .orElse(false); + return finalIsUtf8 || originalIsUtf8 + ? GLOBAL_DISPOSITION_CONTENT_TYPE + : DISPOSITION_CONTENT_TYPE; + } + public BodyPart computeOriginalMessagePart(Message message) throws MessagingException { MimeBodyPart originalMessagePart = new MimeBodyPart(); try { @@ -276,7 +312,7 @@ private Multipart asMime4JMultipart() throws IOException { builder.addBodyPart(BodyPartBuilder.create() .use(new BasicBodyFactory()) .setBody(report.formattedValue(), Charsets.UTF_8) - .setContentType(DISPOSITION_CONTENT_TYPE, UTF_8_CHARSET)); + .setContentType(dispositionContentType(), UTF_8_CHARSET)); return builder.build(); } diff --git a/mdn/src/main/java/org/apache/james/mdn/fields/AddressType.java b/mdn/src/main/java/org/apache/james/mdn/fields/AddressType.java index ccadbc44b90..36e2cbe526a 100644 --- a/mdn/src/main/java/org/apache/james/mdn/fields/AddressType.java +++ b/mdn/src/main/java/org/apache/james/mdn/fields/AddressType.java @@ -26,8 +26,23 @@ public class AddressType { public static final AddressType DNS = new AddressType("dns"); public static final AddressType RFC_822 = new AddressType("rfc822"); + public static final AddressType UTF_8 = new AddressType("utf-8"); public static final AddressType UNKNOWN = new AddressType("unknown"); + /** + * Picks the appropriate addr-type per RFC 6533: {@link #UTF_8} when the + * address contains non-ASCII octets, otherwise {@link #RFC_822}. + */ + public static AddressType pickFor(Text text) { + String value = text.formatted(); + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 0x7F) { + return UTF_8; + } + } + return RFC_822; + } + private final String type; public AddressType(String type) { diff --git a/mdn/src/main/java/org/apache/james/mdn/fields/FinalRecipient.java b/mdn/src/main/java/org/apache/james/mdn/fields/FinalRecipient.java index 0e5ade2a0da..f80914f807d 100644 --- a/mdn/src/main/java/org/apache/james/mdn/fields/FinalRecipient.java +++ b/mdn/src/main/java/org/apache/james/mdn/fields/FinalRecipient.java @@ -57,7 +57,7 @@ public Builder finalRecipient(Text finalRecipient) { public FinalRecipient build() { Preconditions.checkNotNull(finalRecipient); - return new FinalRecipient(addressType.orElse(AddressType.RFC_822), finalRecipient); + return new FinalRecipient(addressType.orElseGet(() -> AddressType.pickFor(finalRecipient)), finalRecipient); } } diff --git a/mdn/src/main/java/org/apache/james/mdn/fields/OriginalRecipient.java b/mdn/src/main/java/org/apache/james/mdn/fields/OriginalRecipient.java index 01ea441cf75..28f987e7026 100644 --- a/mdn/src/main/java/org/apache/james/mdn/fields/OriginalRecipient.java +++ b/mdn/src/main/java/org/apache/james/mdn/fields/OriginalRecipient.java @@ -61,7 +61,7 @@ public Builder originalRecipient(Text originalRecipient) { public OriginalRecipient build() { Preconditions.checkNotNull(originalRecipient); - return new OriginalRecipient(addressType.orElse(AddressType.RFC_822), originalRecipient); + return new OriginalRecipient(addressType.orElseGet(() -> AddressType.pickFor(originalRecipient)), originalRecipient); } } diff --git a/mdn/src/main/scala/org/apache/james/mdn/MDNReportParser.scala b/mdn/src/main/scala/org/apache/james/mdn/MDNReportParser.scala index 84712365447..33c6e9ad7a8 100644 --- a/mdn/src/main/scala/org/apache/james/mdn/MDNReportParser.scala +++ b/mdn/src/main/scala/org/apache/james/mdn/MDNReportParser.scala @@ -111,13 +111,17 @@ class MDNReportParser(val input: ParserInput) extends Parser { private def uaName: Rule0 = rule { zeroOrMore(textNoSemi) } /* text-no-semi = %d1-9 / ; "text" characters excluding NUL, CR, - %d11 / %d12 / %d14-58 / %d60-127 ; LF, or semi-colon */ + %d11 / %d12 / %d14-58 / %d60-127 ; LF, or semi-colon + + RFC 6533 3.2 widens "text" to UTF-8, so anything above US-ASCII is + accepted too (parboiled matches UTF-16 code units; surrogate pairs are + each in that range). */ private def textNoSemi: Rule0 = rule { CharPredicate(1.toChar to 9.toChar) | ch(11) | ch(12) | CharPredicate(14.toChar to 58.toChar) | - CharPredicate(60.toChar to 127.toChar) + CharPredicate.from(c => c >= 60.toChar && c != EOI) } // ua-product = *([FWS] text) @@ -126,12 +130,15 @@ class MDNReportParser(val input: ParserInput) extends Parser { /* text = %d1-9 / ; Characters excluding CR %d11 / ; and LF %d12 / - %d14-127 */ + %d14-127 + + RFC 6533 3.2 widens this to UTF-8 for the global-disposition-notification + form, hence everything above US-ASCII is accepted as well. */ private def text = rule { CharPredicate(1.toChar to 9.toChar) | ch(11) | ch(12) | - CharPredicate(14.toChar to 127.toChar) + CharPredicate.from(c => c >= 14.toChar && c != EOI) } /* OWS = [CFWS] diff --git a/mdn/src/test/java/org/apache/james/mdn/MDNTest.java b/mdn/src/test/java/org/apache/james/mdn/MDNTest.java index 001354fa60d..c598c107865 100644 --- a/mdn/src/test/java/org/apache/james/mdn/MDNTest.java +++ b/mdn/src/test/java/org/apache/james/mdn/MDNTest.java @@ -532,4 +532,86 @@ public void originalMessageShouldBeContainInMimeMessage() throws Exception { private String asString(Message message) throws Exception { return new String(DefaultMessageWriter.asBytes(message), StandardCharsets.UTF_8); } + + // RFC 6533 section + + @Test + void asMime4JMessageShouldUseLegacyContentTypeWhenRecipientsAreAscii() throws Exception { + MDN mdn = MDN.builder() + .humanReadableText("human") + .report(MINIMAL_REPORT) + .build(); + + assertThat(asString(mdn.asMime4JMessageBuilder().build())) + .contains("Content-Type: message/disposition-notification") + .doesNotContain("message/global-disposition-notification"); + } + + @Test + void asMime4JMessageShouldUseGlobalContentTypeWhenFinalRecipientIsUtf8() throws Exception { + MDNReport report = MDNReport.builder() + .finalRecipientField(FinalRecipient.builder() + .finalRecipient(Text.fromRawText("user@grå.org")) + .build()) + .dispositionField(Disposition.builder() + .actionMode(DispositionActionMode.Automatic) + .sendingMode(DispositionSendingMode.Automatic) + .type(DispositionType.Deleted) + .build()) + .build(); + MDN mdn = MDN.builder() + .humanReadableText("human") + .report(report) + .build(); + + assertThat(asString(mdn.asMime4JMessageBuilder().build())) + .contains("Content-Type: message/global-disposition-notification"); + } + + @Test + void parseShouldAcceptGlobalDispositionNotificationContentType() throws Exception { + MDNReport parsed = parseReportWithContentType( + "Final-Recipient: utf-8; user@grå.org\r\n" + + "Disposition: automatic-action/MDN-sent-automatically;processed/error,failed\r\n", + "message/global-disposition-notification"); + + assertThat(parsed.getFinalRecipientField().getFinalRecipient()) + .isEqualTo(Text.fromRawText("user@grå.org")); + assertThat(parsed.getFinalRecipientField().getAddressType()) + .isEqualTo(AddressType.UTF_8); + } + + @Test + void parsedReportShouldBeIndistinguishableAcrossFormatsForAsciiAddress() throws Exception { + // Same recipient, both addr-types — parsed reports should be equal on the + // fields callers actually care about (recipient + disposition). + MDNReport legacy = parseReportWithContentType( + "Final-Recipient: rfc822; user@example.com\r\n" + + "Disposition: automatic-action/MDN-sent-automatically;processed/error,failed\r\n", + "message/disposition-notification"); + MDNReport global = parseReportWithContentType( + "Final-Recipient: rfc822; user@example.com\r\n" + + "Disposition: automatic-action/MDN-sent-automatically;processed/error,failed\r\n", + "message/global-disposition-notification"); + + assertThat(legacy.getFinalRecipientField().getFinalRecipient()) + .isEqualTo(global.getFinalRecipientField().getFinalRecipient()); + assertThat(legacy.getDispositionField()) + .isEqualTo(global.getDispositionField()); + } + + private MDNReport parseReportWithContentType(String body, String contentType) throws Exception { + BodyPart mdnBodyPart = BodyPartBuilder + .create() + .setBody(SingleBodyBuilder.create().setText(body).setCharset(StandardCharsets.UTF_8).buildText()) + .setContentType(contentType) + .build(); + Message message = Message.Builder.of() + .setBody(MultipartBuilder.create("report") + .addTextPart("first", StandardCharsets.UTF_8) + .addBodyPart(mdnBodyPart) + .build()) + .build(); + return MDN.parse(message).getReport(); + } } diff --git a/mdn/src/test/java/org/apache/james/mdn/fields/AddressTypeTest.java b/mdn/src/test/java/org/apache/james/mdn/fields/AddressTypeTest.java index 0b610d5873a..5333d5a6336 100644 --- a/mdn/src/test/java/org/apache/james/mdn/fields/AddressTypeTest.java +++ b/mdn/src/test/java/org/apache/james/mdn/fields/AddressTypeTest.java @@ -86,4 +86,21 @@ void typeShouldBeTrimmed() { assertThat(addressType.getType()) .isEqualTo("ab"); } + + @Test + void utf8ConstantShouldHoldRfc6533Value() { + assertThat(AddressType.UTF_8.getType()).isEqualTo("utf-8"); + } + + @Test + void pickForShouldReturnRfc822ForAsciiAddress() { + assertThat(AddressType.pickFor(Text.fromRawText("user@example.com"))) + .isEqualTo(AddressType.RFC_822); + } + + @Test + void pickForShouldReturnUtf8ForNonAsciiAddress() { + assertThat(AddressType.pickFor(Text.fromRawText("user@grå.org"))) + .isEqualTo(AddressType.UTF_8); + } } diff --git a/mdn/src/test/java/org/apache/james/mdn/fields/FinalRecipientTest.java b/mdn/src/test/java/org/apache/james/mdn/fields/FinalRecipientTest.java index bb8c84d5983..b18a4d347d0 100644 --- a/mdn/src/test/java/org/apache/james/mdn/fields/FinalRecipientTest.java +++ b/mdn/src/test/java/org/apache/james/mdn/fields/FinalRecipientTest.java @@ -72,6 +72,27 @@ void typeShouldDefaultToRfc822() { .build()); } + @Test + void typeShouldDefaultToUtf8WhenAddressContainsNonAscii() { + // RFC 6533 §3.1: use the utf-8 addr-type when the address contains UTF-8 + Text address = Text.fromRawText("arnt@grå.org"); + + assertThat(FinalRecipient.builder() + .finalRecipient(address) + .build() + .getAddressType()) + .isEqualTo(AddressType.UTF_8); + } + + @Test + void formattedValueShouldDisplayUtf8TypeForNonAsciiAddress() { + assertThat(FinalRecipient.builder() + .finalRecipient(Text.fromRawText("arnt@grå.org")) + .build() + .formattedValue()) + .isEqualTo("Final-Recipient: utf-8; arnt@grå.org"); + } + @Test void formattedValueShouldDisplayAddress() { assertThat(FinalRecipient.builder() diff --git a/mdn/src/test/java/org/apache/james/mdn/fields/OriginalRecipientTest.java b/mdn/src/test/java/org/apache/james/mdn/fields/OriginalRecipientTest.java index 964132e0ca4..ae78aac2bd2 100644 --- a/mdn/src/test/java/org/apache/james/mdn/fields/OriginalRecipientTest.java +++ b/mdn/src/test/java/org/apache/james/mdn/fields/OriginalRecipientTest.java @@ -72,6 +72,17 @@ void addressTypeShouldDefaultToRfc822() { .build()); } + @Test + void addressTypeShouldDefaultToUtf8WhenAddressContainsNonAscii() { + Text address = Text.fromRawText("arnt@grå.org"); + + assertThat(OriginalRecipient.builder() + .originalRecipient(address) + .build() + .getAddressType()) + .isEqualTo(AddressType.UTF_8); + } + @Test void formattedValueShouldDisplayAddress() { assertThat(OriginalRecipient.builder() diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_starts_with_starttls.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_starts_with_starttls.test index 88c656f7db9..41cc52c569e 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_starts_with_starttls.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_starts_with_starttls.test @@ -9,6 +9,7 @@ S: 250-AUTH=LOGIN PLAIN S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES S: 250-8BITMIME +S: 250-SMTPUTF8 S: 250 STARTTLS S: 250 2.1.0 Sender OK S: 250 2.1.5 Recipient OK diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_with_starttls.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_with_starttls.test index c25db1be377..7480678a126 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_with_starttls.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/data_with_starttls.test @@ -9,6 +9,7 @@ S: 250-AUTH=LOGIN PLAIN S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES S: 250-8BITMIME +S: 250-SMTPUTF8 S: 250 STARTTLS S: 250 2.1.0 Sender OK S: 250 2.1.5 Recipient OK diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/helo.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/helo.test index 36d609e1183..85cab828359 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/helo.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/helo.test @@ -6,7 +6,8 @@ C: data S: 250.* S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES -S: 250 8BITMIME +S: 250-8BITMIME +S: 250 SMTPUTF8 S: 250 2.1.0 Sender OK S: 250 2.1.5 Recipient OK S: 354 Ok Send data ending with . diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/rcpt_with_starttls.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/rcpt_with_starttls.test index 5b6de69b98a..5d570de829c 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/rcpt_with_starttls.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/rcpt_with_starttls.test @@ -9,6 +9,7 @@ S: 250-AUTH=LOGIN PLAIN S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES S: 250-8BITMIME +S: 250-SMTPUTF8 S: 250 STARTTLS S: 250 2.1.0 Sender OK S: 250 2.1.5 Recipient OK diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls.test index d71186b47d8..724a1bd54d7 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls.test @@ -7,6 +7,7 @@ S: 250-AUTH=LOGIN PLAIN S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES S: 250-8BITMIME +S: 250-SMTPUTF8 S: 250 STARTTLS C: starttls diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_session_fixation.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_session_fixation.test index 2d1b1e562ac..bd756b06dad 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_session_fixation.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_session_fixation.test @@ -7,6 +7,7 @@ S: 250-AUTH=LOGIN PLAIN S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES S: 250-8BITMIME +S: 250-SMTPUTF8 S: 250 STARTTLS C: AUTH LOGIN diff --git a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_with_injection.test b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_with_injection.test index 90068f635df..e9ce1c1ad48 100644 --- a/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_with_injection.test +++ b/mpt/impl/smtp/core/src/main/resources/org/apache/james/smtp/scripts/starttls_with_injection.test @@ -7,6 +7,7 @@ S: 250-AUTH=LOGIN PLAIN S: 250-PIPELINING S: 250-ENHANCEDSTATUSCODES S: 250-8BITMIME +S: 250-SMTPUTF8 S: 250 STARTTLS C: starttls\r\nmail from:\r\n diff --git a/protocols/api/src/main/java/org/apache/james/protocols/api/AbstractProtocolTransport.java b/protocols/api/src/main/java/org/apache/james/protocols/api/AbstractProtocolTransport.java index a3ef7eb8f4f..fa4f63d2aa6 100644 --- a/protocols/api/src/main/java/org/apache/james/protocols/api/AbstractProtocolTransport.java +++ b/protocols/api/src/main/java/org/apache/james/protocols/api/AbstractProtocolTransport.java @@ -85,7 +85,12 @@ protected static byte[] toBytes(Response response) { builder.append(CRLF); } } - return builder.toString().getBytes(StandardCharsets.US_ASCII); + // RFC 6531 §3.7.4.2: when a server echoes a UTF-8 mailbox address back + // to the client, those octets are UTF-8; all other reply content stays + // ASCII. UTF-8 is a strict superset of ASCII, so encoding the whole + // reply in UTF-8 preserves ASCII-only replies byte-for-byte while + // allowing non-ASCII addresses to survive the echo. + return builder.toString().getBytes(StandardCharsets.UTF_8); } /** diff --git a/protocols/api/src/main/java/org/apache/james/protocols/api/ProtocolSessionImpl.java b/protocols/api/src/main/java/org/apache/james/protocols/api/ProtocolSessionImpl.java index 155fe481b5f..2765add7f00 100644 --- a/protocols/api/src/main/java/org/apache/james/protocols/api/ProtocolSessionImpl.java +++ b/protocols/api/src/main/java/org/apache/james/protocols/api/ProtocolSessionImpl.java @@ -19,10 +19,9 @@ package org.apache.james.protocols.api; -import static java.nio.charset.StandardCharsets.US_ASCII; - import java.net.InetSocketAddress; import java.nio.charset.Charset; +import java.nio.charset.StandardCharsets; import java.time.Duration; import java.util.HashMap; import java.util.Map; @@ -217,7 +216,7 @@ public Optional getAttachment(AttachmentKey key, State state) { */ @Override public Charset getCharset() { - return US_ASCII; + return StandardCharsets.UTF_8; } /** diff --git a/protocols/imap/src/main/java/org/apache/james/imap/api/ImapConstants.java b/protocols/imap/src/main/java/org/apache/james/imap/api/ImapConstants.java index 5619cf290c1..23f9f0db2c1 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/api/ImapConstants.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/api/ImapConstants.java @@ -104,7 +104,9 @@ public interface ImapConstants { Capability SUPPORTS_UIDPLUS = Capability.of("UIDPLUS"); Capability SUPPORTS_ANNOTATION = Capability.of("METADATA"); - + + Capability SUPPORTS_UTF8_ACCEPT = Capability.of("UTF8=ACCEPT"); + String INBOX_NAME = "INBOX"; String MIME_TYPE_TEXT = "TEXT"; diff --git a/protocols/imap/src/main/java/org/apache/james/imap/api/process/ImapSession.java b/protocols/imap/src/main/java/org/apache/james/imap/api/process/ImapSession.java index 791e8ce69a5..26462849b06 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/api/process/ImapSession.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/api/process/ImapSession.java @@ -90,6 +90,8 @@ public String toString() { String MAILBOX_SESSION_ATTRIBUTE_SESSION_KEY = "org.apache.james.api.imap.MAILBOX_SESSION_ATTRIBUTE_SESSION_KEY"; + String UTF8_ENABLED_ATTRIBUTE_SESSION_KEY = "org.apache.james.api.imap.UTF8_ENABLED"; + /** * @return a unique identifier for this session. * @@ -274,6 +276,20 @@ default MailboxSession getMailboxSession() { return (MailboxSession) getAttribute(MAILBOX_SESSION_ATTRIBUTE_SESSION_KEY); } + /** + * Record that the client has ENABLEd UTF8=ACCEPT (RFC 6855). From then on + * mailbox names and strings are UTF-8 octets in both directions, so the + * decoder and the encoder built for this session are configured from + * {@link #utf8Enabled()}. + */ + default void enableUtf8() { + setAttribute(UTF8_ENABLED_ATTRIBUTE_SESSION_KEY, Boolean.TRUE); + } + + default boolean utf8Enabled() { + return Boolean.TRUE.equals(getAttribute(UTF8_ENABLED_ATTRIBUTE_SESSION_KEY)); + } + default Username getUserName() { return Optional.ofNullable(getMailboxSession()) .map(MailboxSession::getUser) diff --git a/protocols/imap/src/main/java/org/apache/james/imap/decode/ImapRequestLineReader.java b/protocols/imap/src/main/java/org/apache/james/imap/decode/ImapRequestLineReader.java index 3aa5bbbe185..42c52689e12 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/decode/ImapRequestLineReader.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/decode/ImapRequestLineReader.java @@ -19,7 +19,7 @@ package org.apache.james.imap.decode; -import static java.nio.charset.StandardCharsets.US_ASCII; +import static java.nio.charset.StandardCharsets.UTF_8; import java.io.Closeable; import java.io.IOException; @@ -291,6 +291,7 @@ public static boolean isQuotedSpecial(char chr) { protected char nextChar; // unknown protected boolean nextSeen = false; + private boolean utf8Accept = false; private final StringBuilder stringBuilder = new StringBuilder(); /** @@ -489,7 +490,24 @@ public String nstring() throws DecodingException { * */ public String mailbox() throws DecodingException { - return ModifiedUtf7.decodeModifiedUTF7(mailboxUTF7()); + if (utf8Accept) { + String mailbox = astring(UTF_8); + if (mailbox.equalsIgnoreCase(ImapConstants.INBOX_NAME)) { + return ImapConstants.INBOX_NAME; + } + return mailbox; + } + return ModifiedUtf7.decodeModifiedUTF7(mailboxUTF7()); + } + + /** + * When set, {@link #mailbox()} treats the astring as UTF-8 and does not + * run Modified UTF-7 decoding. Callers should set this from + * {@code ImapSession#utf8Enabled()} once the session state is known. + */ + public ImapRequestLineReader setUtf8Accept(boolean utf8Accept) { + this.utf8Accept = utf8Accept; + return this; } /** @@ -501,7 +519,8 @@ public String mailbox() throws DecodingException { * variants of ;; INBOX (e.g. "iNbOx") MUST be interpreted as INBOX ;; not * as an astring. * - * Be aware that mailbox names are encoded via a modified UTF7. For more information RFC3501 + * Be aware that mailbox names are encoded via a modified UTF7 in unextended + * IMAP. For more information see RFC3501. RFC9755 changes this. */ public String mailboxUTF7() throws DecodingException { String mailbox = astring(); @@ -612,11 +631,15 @@ private static boolean isWhitespace(char next) { * this method. * * @param charset - * , or null for US-ASCII + * , or null for UTF-8 */ public String consumeLiteral(Charset charset) throws DecodingException { if (charset == null) { - return consumeLiteral(US_ASCII); + // RFC 9051/6855: literals carry UTF-8 octets once UTF8=ACCEPT is + // enabled. Accept them unconditionally, as consumeQuoted() does: + // UTF-8 is a superset of US-ASCII, so unextended sessions are + // unaffected except that 8-bit octets are no longer rejected. + return consumeLiteral(UTF_8); } else { try { ImmutablePair literal = consumeLiteral(false); @@ -733,7 +756,7 @@ public String consumeQuoted() throws DecodingException { */ protected String consumeQuoted(Charset charset) throws DecodingException { if (charset == null) { - return consumeQuoted(US_ASCII); + return consumeQuoted(UTF_8); } else { // The 1st character must be '"' consumeChar('"'); diff --git a/protocols/imap/src/main/java/org/apache/james/imap/encode/base/ImapResponseComposerImpl.java b/protocols/imap/src/main/java/org/apache/james/imap/encode/base/ImapResponseComposerImpl.java index dff62000a6f..1ce6bda6576 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/encode/base/ImapResponseComposerImpl.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/encode/base/ImapResponseComposerImpl.java @@ -22,6 +22,7 @@ import static java.nio.charset.StandardCharsets.US_ASCII; import java.io.IOException; +import java.nio.charset.StandardCharsets; import java.util.ArrayList; import java.util.List; import java.util.Optional; @@ -68,6 +69,8 @@ public class ImapResponseComposerImpl implements ImapConstants, ImapResponseComp private boolean skipNextSpace; + private boolean utf8Accepted = false; + // Text chunks and literals gathered to be emitted as a single SequencedLiteral (one flush, no copy). Null until a literal is buffered. private List pendingLiteralParts; @@ -260,7 +263,21 @@ public ImapResponseComposer message(long number) throws IOException { @Override public ImapResponseComposer mailbox(String mailboxName) throws IOException { - quote(ModifiedUtf7.encodeModifiedUTF7(mailboxName)); + if (utf8Accepted) { + quote(mailboxName); + } else { + quote(ModifiedUtf7.encodeModifiedUTF7(mailboxName)); + } + return this; + } + + /** + * Per RFC 9755, when the client has ENABLEd UTF8=ACCEPT the server emits + * mailbox names and other strings as UTF-8 octets (not Modified UTF-7). + * Set this once per composer, from {@code ImapSession#utf8Enabled()}. + */ + public ImapResponseComposerImpl setUtf8Accepted(boolean utf8Accepted) { + this.utf8Accepted = utf8Accepted; return this; } @@ -272,19 +289,27 @@ public ImapResponseComposer commandName(ImapCommand command) throws IOException @Override public ImapResponseComposer quote(String message) throws IOException { space(); - final int length = message.length(); - buffer.write(BYTE_DQUOTE); - for (int i = 0; i < length; i++) { - char character = message.charAt(i); - if (character == ImapConstants.BACK_SLASH || character == DQUOTE) { - buffer.write(BYTE_BACK_SLASH); + if (utf8Accepted) { + for (byte b : message.getBytes(StandardCharsets.UTF_8)) { + if (b == BYTE_BACK_SLASH || b == BYTE_DQUOTE) { + buffer.write(BYTE_BACK_SLASH); + } + buffer.write(b); } - // 7-bit ASCII only - if (character >= 128) { - buffer.write(BYTE_QUESTION); - } else { - buffer.write((byte) character); + } else { + final int length = message.length(); + for (int i = 0; i < length; i++) { + char character = message.charAt(i); + if (character == ImapConstants.BACK_SLASH || character == DQUOTE) { + buffer.write(BYTE_BACK_SLASH); + } + // 7-bit ASCII only + if (character >= 128) { + buffer.write(BYTE_QUESTION); + } else { + buffer.write((byte) character); + } } } buffer.write(BYTE_DQUOTE); diff --git a/protocols/imap/src/main/java/org/apache/james/imap/processor/CapabilityProcessor.java b/protocols/imap/src/main/java/org/apache/james/imap/processor/CapabilityProcessor.java index 99033730816..088588d0b4a 100644 --- a/protocols/imap/src/main/java/org/apache/james/imap/processor/CapabilityProcessor.java +++ b/protocols/imap/src/main/java/org/apache/james/imap/processor/CapabilityProcessor.java @@ -26,6 +26,7 @@ import static org.apache.james.imap.api.ImapConstants.SUPPORTS_OBJECTID; import static org.apache.james.imap.api.ImapConstants.SUPPORTS_RFC3348; import static org.apache.james.imap.api.ImapConstants.SUPPORTS_SAVEDATE; +import static org.apache.james.imap.api.ImapConstants.SUPPORTS_UTF8_ACCEPT; import static org.apache.james.mailbox.MailboxManager.MessageCapabilities.UniqueID; import java.util.ArrayList; @@ -36,6 +37,7 @@ import jakarta.inject.Inject; import org.apache.james.imap.api.ImapConfiguration; +import org.apache.james.imap.api.ImapMessage; import org.apache.james.imap.api.message.Capability; import org.apache.james.imap.api.message.response.StatusResponseFactory; import org.apache.james.imap.api.process.ImapSession; @@ -49,7 +51,7 @@ import reactor.core.publisher.Mono; -public class CapabilityProcessor extends AbstractMailboxProcessor implements CapabilityImplementingProcessor { +public class CapabilityProcessor extends AbstractMailboxProcessor implements CapabilityImplementingProcessor, PermitEnableCapabilityProcessor { private static final List CAPS = ImmutableList.of( BASIC_CAPABILITIES, @@ -58,8 +60,13 @@ public class CapabilityProcessor extends AbstractMailboxProcessor ENABLEABLE_CAPS = ImmutableList.of(SUPPORTS_UTF8_ACCEPT); + private final List capabilities = new ArrayList<>(); private final Set disabledCaps = new HashSet<>(); @@ -106,6 +113,19 @@ public void addProcessor(CapabilityImplementingProcessor implementor) { public List getImplementedCapabilities(ImapSession session) { return CAPS; } + + @Override + public List getPermitEnableCapabilities(ImapSession session) { + return ENABLEABLE_CAPS; + } + + @Override + public Mono enable(ImapMessage message, Responder responder, ImapSession session, Capability capability) { + if (SUPPORTS_UTF8_ACCEPT.equals(capability)) { + session.enableUtf8(); + } + return Mono.empty(); + } /** * Return all supported CAPABILITIES for this {@link ImapSession} diff --git a/protocols/imap/src/test/java/org/apache/james/imap/decode/ImapRequestLineReaderTest.java b/protocols/imap/src/test/java/org/apache/james/imap/decode/ImapRequestLineReaderTest.java index b2bbdd31888..2aad9c33538 100644 --- a/protocols/imap/src/test/java/org/apache/james/imap/decode/ImapRequestLineReaderTest.java +++ b/protocols/imap/src/test/java/org/apache/james/imap/decode/ImapRequestLineReaderTest.java @@ -57,4 +57,51 @@ void nextNonSpaceCharShouldThrowExceptionWhenNotFound() { assertThatThrownBy(() -> lineReader.nextNonSpaceChar()).isInstanceOf(DecodingException.class); } + + @Test + void mailboxShouldDecodeModifiedUtf7WhenUtf8AcceptNotEnabled() throws Exception { + // Wire form "a&--b" is the Modified UTF-7 encoding of "a&-b". + inputStream = new ByteArrayInputStream("\"a&--b\" ".getBytes(StandardCharsets.US_ASCII)); + lineReader = new ImapRequestStreamLineReader(inputStream, outputStream); + + assertThat(lineReader.mailbox()).isEqualTo("a&-b"); + } + + @Test + void mailboxShouldDecodeUnicodeModifiedUtf7WhenUtf8AcceptNotEnabled() throws Exception { + // Wire form "gr&AOU-" is the Modified UTF-7 encoding of "grå". + inputStream = new ByteArrayInputStream("\"gr&AOU-\" ".getBytes(StandardCharsets.US_ASCII)); + lineReader = new ImapRequestStreamLineReader(inputStream, outputStream); + + assertThat(lineReader.mailbox()).isEqualTo("grå"); + } + + @Test + void mailboxShouldReturnRawStringWhenUtf8AcceptEnabledAndNameContainsAmpersand() throws Exception { + inputStream = new ByteArrayInputStream("\"a&-b\" ".getBytes(StandardCharsets.UTF_8)); + lineReader = new ImapRequestStreamLineReader(inputStream, outputStream); + lineReader.setUtf8Accept(true); + + assertThat(lineReader.mailbox()).isEqualTo("a&-b"); + } + + @Test + void mailboxShouldReturnRawUnicodeWhenUtf8AcceptEnabled() throws Exception { + inputStream = new ByteArrayInputStream("\"grå\" ".getBytes(StandardCharsets.UTF_8)); + lineReader = new ImapRequestStreamLineReader(inputStream, outputStream); + lineReader.setUtf8Accept(true); + + assertThat(lineReader.mailbox()).isEqualTo("grå"); + } + + @Test + void astringShouldDecodeUtf8QuotedStringByDefault() throws Exception { + // Many IMAP clients put UTF-8 in quoted-string arguments (e.g. + // SEARCH HEADER Subject "grå") without an explicit CHARSET. RFC 9051 + // allows this, and we accept it regardless of UTF8=ACCEPT. + inputStream = new ByteArrayInputStream("\"grå\" ".getBytes(StandardCharsets.UTF_8)); + lineReader = new ImapRequestStreamLineReader(inputStream, outputStream); + + assertThat(lineReader.astring()).isEqualTo("grå"); + } } \ No newline at end of file diff --git a/protocols/imap/src/test/java/org/apache/james/imap/processor/CapabilityProcessorTest.java b/protocols/imap/src/test/java/org/apache/james/imap/processor/CapabilityProcessorTest.java index 1b89404ba5a..1555a694ba9 100644 --- a/protocols/imap/src/test/java/org/apache/james/imap/processor/CapabilityProcessorTest.java +++ b/protocols/imap/src/test/java/org/apache/james/imap/processor/CapabilityProcessorTest.java @@ -78,4 +78,18 @@ void condstoreShouldBeNotSupportedByDefault() { Set supportedCapabilities = testee.getSupportedCapabilities(null); assertThat(supportedCapabilities).doesNotContain(ImapConstants.SUPPORTS_CONDSTORE); } + + @Test + void utf8AcceptShouldBeAdvertised() { + testee.configure(ImapConfiguration.builder().build()); + + Set supportedCapabilities = testee.getSupportedCapabilities(null); + assertThat(supportedCapabilities).contains(ImapConstants.SUPPORTS_UTF8_ACCEPT); + } + + @Test + void utf8AcceptShouldBeEnableable() { + assertThat(testee.getPermitEnableCapabilities(null)) + .contains(ImapConstants.SUPPORTS_UTF8_ACCEPT); + } } diff --git a/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPSServerTest.java b/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPSServerTest.java index 87e04a29f13..6c873941644 100644 --- a/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPSServerTest.java +++ b/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPSServerTest.java @@ -1,103 +1,113 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.protocols.lmtp; - -import java.io.IOException; -import java.util.ArrayList; -import java.util.List; - -import javax.net.ssl.SSLContext; - -import org.apache.commons.net.smtp.RelayPath; -import org.apache.commons.net.smtp.SMTPClient; -import org.apache.commons.net.smtp.SMTPReply; -import org.apache.commons.net.smtp.SMTPSClient; -import org.apache.james.protocols.api.utils.BogusSslContextFactory; -import org.apache.james.protocols.api.utils.BogusTrustManagerFactory; - -public abstract class AbstractLMTPSServerTest extends AbstractLMTPServerTest { - - @Override - protected SMTPClient createClient() { - LMTPSClient client = new LMTPSClient(true, BogusSslContextFactory.getClientContext()); - client.setTrustManager(BogusTrustManagerFactory.getTrustManagers()[0]); - - return client; - } - - protected final class LMTPSClient extends SMTPSClient implements LMTPClient { - - private final List replies = new ArrayList<>(); - private int rcptCount = 0; - - - public LMTPSClient(boolean implicit, SSLContext ctx) { - super(implicit, ctx); - } - - - @Override - public boolean addRecipient(String address) throws IOException { - boolean ok = super.addRecipient(address); - if (ok) { - rcptCount++; - } - return ok; - } - - @Override - public boolean addRecipient(RelayPath path) throws IOException { - boolean ok = super.addRecipient(path); - if (ok) { - rcptCount++; - } - return ok; - } - - /** - * Issue the LHLO command - */ - @Override - public int helo(String hostname) throws IOException { - return sendCommand("LHLO", hostname); - } - - @Override - public int[] getReplies() throws IOException { - int[] codes = new int[replies.size()]; - for (int i = 0; i < codes.length; i++) { - codes[i] = replies.remove(0); - } - return codes; - } - - @Override - public boolean completePendingCommand() throws IOException { - for (int i = 0; i < rcptCount; i++) { - replies.add(getReply()); - } - - return replies.stream() - .mapToInt(code -> code) - .anyMatch(SMTPReply::isPositiveCompletion); - } - - - } -} +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.protocols.lmtp; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; +import java.util.ArrayList; +import java.util.List; + +import javax.net.ssl.SSLContext; + +import org.apache.commons.net.smtp.RelayPath; +import org.apache.commons.net.smtp.SMTPClient; +import org.apache.commons.net.smtp.SMTPReply; +import org.apache.commons.net.smtp.SMTPSClient; +import org.apache.james.protocols.api.utils.BogusSslContextFactory; +import org.apache.james.protocols.api.utils.BogusTrustManagerFactory; + +public abstract class AbstractLMTPSServerTest extends AbstractLMTPServerTest { + + /** See {@code AbstractSMTPSServerTest}: the raw exchanges need a TLS socket here. */ + @Override + protected Socket createRawSocket(InetSocketAddress address) throws IOException { + return BogusSslContextFactory.getClientContext() + .getSocketFactory() + .createSocket(address.getAddress().getHostAddress(), address.getPort()); + } + + @Override + protected SMTPClient createClient() { + LMTPSClient client = new LMTPSClient(true, BogusSslContextFactory.getClientContext()); + client.setTrustManager(BogusTrustManagerFactory.getTrustManagers()[0]); + + return client; + } + + protected final class LMTPSClient extends SMTPSClient implements LMTPClient { + + private final List replies = new ArrayList<>(); + private int rcptCount = 0; + + + public LMTPSClient(boolean implicit, SSLContext ctx) { + super(implicit, ctx); + } + + + @Override + public boolean addRecipient(String address) throws IOException { + boolean ok = super.addRecipient(address); + if (ok) { + rcptCount++; + } + return ok; + } + + @Override + public boolean addRecipient(RelayPath path) throws IOException { + boolean ok = super.addRecipient(path); + if (ok) { + rcptCount++; + } + return ok; + } + + /** + * Issue the LHLO command + */ + @Override + public int helo(String hostname) throws IOException { + return sendCommand("LHLO", hostname); + } + + @Override + public int[] getReplies() throws IOException { + int[] codes = new int[replies.size()]; + for (int i = 0; i < codes.length; i++) { + codes[i] = replies.remove(0); + } + return codes; + } + + @Override + public boolean completePendingCommand() throws IOException { + for (int i = 0; i < rcptCount; i++) { + replies.add(getReply()); + } + + return replies.stream() + .mapToInt(code -> code) + .anyMatch(SMTPReply::isPositiveCompletion); + } + + + } +} diff --git a/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPServerTest.java b/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPServerTest.java index 50820a1ef0d..291d6a4d0ed 100644 --- a/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPServerTest.java +++ b/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/AbstractLMTPServerTest.java @@ -1,344 +1,349 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.protocols.lmtp; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.IOException; -import java.net.InetSocketAddress; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - -import org.apache.commons.net.smtp.RelayPath; -import org.apache.commons.net.smtp.SMTPClient; -import org.apache.commons.net.smtp.SMTPReply; -import org.apache.james.core.MailAddress; -import org.apache.james.protocols.api.Protocol; -import org.apache.james.protocols.api.ProtocolServer; -import org.apache.james.protocols.api.handler.ProtocolHandler; -import org.apache.james.protocols.api.handler.WiringException; -import org.apache.james.protocols.api.utils.ProtocolServerUtils; -import org.apache.james.protocols.lmtp.hook.DeliverToRecipientHook; -import org.apache.james.protocols.smtp.AbstractSMTPServerTest; -import org.apache.james.protocols.smtp.MailEnvelope; -import org.apache.james.protocols.smtp.SMTPProtocol; -import org.apache.james.protocols.smtp.SMTPSession; -import org.apache.james.protocols.smtp.hook.HookResult; -import org.apache.james.protocols.smtp.hook.MessageHook; -import org.apache.james.protocols.smtp.utils.TestMessageHook; -import org.junit.Ignore; -import org.junit.jupiter.api.Test; - -public abstract class AbstractLMTPServerTest extends AbstractSMTPServerTest { - - @Override - protected Protocol createProtocol(ProtocolHandler... handlers) throws WiringException { - LMTPProtocolHandlerChain chain = new LMTPProtocolHandlerChain(); - List hList = new ArrayList<>(); - - for (ProtocolHandler handler : handlers) { - if (handler instanceof MessageHook) { - handler = new MessageHookAdapter((MessageHook) handler); - } - hList.add(handler); - } - chain.addAll(0, hList); - chain.wireExtensibleHandlers(); - return new SMTPProtocol(chain, new LMTPConfigurationImpl()); - } - - - @Ignore("LMTP can't handle the queue") - @Override - protected void testDeliveryWith4SimultaneousThreads() { - } - - @Ignore("Disable") - @Override - protected void testInvalidNoBracketsEnformance() throws Exception { - } - - - @Ignore("Disable") - @Override - protected void testHeloEnforcement() throws Exception { - } - - - @Ignore("Disable") - @Override - public void testHeloEnforcementDisabled() throws Exception { - } - - - @Override - protected void testMailWithoutBrackets() throws Exception { - TestMessageHook hook = new TestMessageHook(); - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.mail(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Override - protected void testRcptWithoutBrackets() throws Exception { - TestMessageHook hook = new TestMessageHook(); - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - - client.rcpt(RCPT1); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - } - - - @Test - protected void testEhloNotSupported() throws Exception { - TestMessageHook hook = new TestMessageHook(); - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.sendCommand("HELO localhost"); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - } - - @Test - void testDeliveryHook() throws Exception { - TestDeliverHook deliverHook = new TestDeliverHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(deliverHook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT2); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - - assertThat(client.sendShortMessageData(MSG1)).isTrue(); - - int[] replies = ((LMTPClient)client).getReplies(); - - assertThat(replies.length).describedAs("Expected two replies").isEqualTo(2); - - assertThat(SMTPReply.isNegativePermanent(replies[0])).isTrue(); - assertThat(SMTPReply.isPositiveCompletion(replies[1])).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = deliverHook.getDelivered().iterator(); - assertThat(queued.hasNext()).isTrue(); - - MailEnvelope env = queued.next(); - checkEnvelope(env, SENDER, Arrays.asList(RCPT1, RCPT2), MSG1); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Override - protected SMTPClient createClient() { - return new LMTPClientImpl(); - } - - private final class LMTPClientImpl extends SMTPClient implements LMTPClient { - - private final List replies = new ArrayList<>(); - private int rcptCount = 0; - - - @Override - public boolean addRecipient(String address) throws IOException { - boolean ok = super.addRecipient(address); - if (ok) { - rcptCount++; - } - return ok; - } - - @Override - public boolean addRecipient(RelayPath path) throws IOException { - boolean ok = super.addRecipient(path); - if (ok) { - rcptCount++; - } - return ok; - } - - /** - * Issue the LHLO command - */ - @Override - public int helo(String hostname) throws IOException { - return sendCommand("LHLO", hostname); - } - - @Override - public int[] getReplies() throws IOException { - int[] codes = new int[replies.size()]; - for (int i = 0; i < codes.length; i++) { - codes[i] = replies.remove(0); - } - return codes; - } - - @Override - public boolean completePendingCommand() throws IOException { - for (int i = 0; i < rcptCount; i++) { - replies.add(getReply()); - } - - return replies.stream() - .mapToInt(code -> code) - .anyMatch(SMTPReply::isPositiveCompletion); - } - - - } - - private final class MessageHookAdapter implements DeliverToRecipientHook { - - private final MessageHook hook; - private HookResult result; - - public MessageHookAdapter(MessageHook hook) { - this.hook = hook; - } - - @Override - public HookResult deliver(SMTPSession session, MailAddress recipient, MailEnvelope envelope) { - if (result == null) { - result = hook.onMessage(session, envelope); - } - return result; - } - } - - private final class TestDeliverHook implements DeliverToRecipientHook { - - private final List delivered = new ArrayList<>(); - - @Override - public HookResult deliver(SMTPSession session, MailAddress recipient, MailEnvelope envelope) { - if (RCPT1.equals(recipient.toString())) { - return HookResult.DENY; - } else { - delivered.add(envelope); - return HookResult.OK; - } - } - - public List getDelivered() { - return delivered; - } - } - -} +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.protocols.lmtp; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.net.smtp.RelayPath; +import org.apache.commons.net.smtp.SMTPClient; +import org.apache.commons.net.smtp.SMTPReply; +import org.apache.james.core.MailAddress; +import org.apache.james.protocols.api.Protocol; +import org.apache.james.protocols.api.ProtocolServer; +import org.apache.james.protocols.api.handler.ProtocolHandler; +import org.apache.james.protocols.api.handler.WiringException; +import org.apache.james.protocols.api.utils.ProtocolServerUtils; +import org.apache.james.protocols.lmtp.hook.DeliverToRecipientHook; +import org.apache.james.protocols.smtp.AbstractSMTPServerTest; +import org.apache.james.protocols.smtp.MailEnvelope; +import org.apache.james.protocols.smtp.SMTPProtocol; +import org.apache.james.protocols.smtp.SMTPSession; +import org.apache.james.protocols.smtp.hook.HookResult; +import org.apache.james.protocols.smtp.hook.MessageHook; +import org.apache.james.protocols.smtp.utils.TestMessageHook; +import org.junit.Ignore; +import org.junit.jupiter.api.Test; + +public abstract class AbstractLMTPServerTest extends AbstractSMTPServerTest { + + @Override + protected String greetingCommand() { + return "LHLO"; + } + + @Override + protected Protocol createProtocol(ProtocolHandler... handlers) throws WiringException { + LMTPProtocolHandlerChain chain = new LMTPProtocolHandlerChain(); + List hList = new ArrayList<>(); + + for (ProtocolHandler handler : handlers) { + if (handler instanceof MessageHook) { + handler = new MessageHookAdapter((MessageHook) handler); + } + hList.add(handler); + } + chain.addAll(0, hList); + chain.wireExtensibleHandlers(); + return new SMTPProtocol(chain, new LMTPConfigurationImpl()); + } + + + @Ignore("LMTP can't handle the queue") + @Override + protected void testDeliveryWith4SimultaneousThreads() { + } + + @Ignore("Disable") + @Override + protected void testInvalidNoBracketsEnformance() throws Exception { + } + + + @Ignore("Disable") + @Override + protected void testHeloEnforcement() throws Exception { + } + + + @Ignore("Disable") + @Override + public void testHeloEnforcementDisabled() throws Exception { + } + + + @Override + protected void testMailWithoutBrackets() throws Exception { + TestMessageHook hook = new TestMessageHook(); + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.mail(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Override + protected void testRcptWithoutBrackets() throws Exception { + TestMessageHook hook = new TestMessageHook(); + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + + client.rcpt(RCPT1); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + } + + + @Test + protected void testEhloNotSupported() throws Exception { + TestMessageHook hook = new TestMessageHook(); + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.sendCommand("HELO localhost"); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void testDeliveryHook() throws Exception { + TestDeliverHook deliverHook = new TestDeliverHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(deliverHook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT2); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + + assertThat(client.sendShortMessageData(MSG1)).isTrue(); + + int[] replies = ((LMTPClient)client).getReplies(); + + assertThat(replies.length).describedAs("Expected two replies").isEqualTo(2); + + assertThat(SMTPReply.isNegativePermanent(replies[0])).isTrue(); + assertThat(SMTPReply.isPositiveCompletion(replies[1])).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).describedAs("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = deliverHook.getDelivered().iterator(); + assertThat(queued.hasNext()).isTrue(); + + MailEnvelope env = queued.next(); + checkEnvelope(env, SENDER, Arrays.asList(RCPT1, RCPT2), MSG1); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Override + protected SMTPClient createClient() { + return new LMTPClientImpl(); + } + + private final class LMTPClientImpl extends SMTPClient implements LMTPClient { + + private final List replies = new ArrayList<>(); + private int rcptCount = 0; + + + @Override + public boolean addRecipient(String address) throws IOException { + boolean ok = super.addRecipient(address); + if (ok) { + rcptCount++; + } + return ok; + } + + @Override + public boolean addRecipient(RelayPath path) throws IOException { + boolean ok = super.addRecipient(path); + if (ok) { + rcptCount++; + } + return ok; + } + + /** + * Issue the LHLO command + */ + @Override + public int helo(String hostname) throws IOException { + return sendCommand("LHLO", hostname); + } + + @Override + public int[] getReplies() throws IOException { + int[] codes = new int[replies.size()]; + for (int i = 0; i < codes.length; i++) { + codes[i] = replies.remove(0); + } + return codes; + } + + @Override + public boolean completePendingCommand() throws IOException { + for (int i = 0; i < rcptCount; i++) { + replies.add(getReply()); + } + + return replies.stream() + .mapToInt(code -> code) + .anyMatch(SMTPReply::isPositiveCompletion); + } + + + } + + private final class MessageHookAdapter implements DeliverToRecipientHook { + + private final MessageHook hook; + private HookResult result; + + public MessageHookAdapter(MessageHook hook) { + this.hook = hook; + } + + @Override + public HookResult deliver(SMTPSession session, MailAddress recipient, MailEnvelope envelope) { + if (result == null) { + result = hook.onMessage(session, envelope); + } + return result; + } + } + + private final class TestDeliverHook implements DeliverToRecipientHook { + + private final List delivered = new ArrayList<>(); + + @Override + public HookResult deliver(SMTPSession session, MailAddress recipient, MailEnvelope envelope) { + if (RCPT1.equals(recipient.toString())) { + return HookResult.DENY; + } else { + delivered.add(envelope); + return HookResult.OK; + } + } + + public List getDelivered() { + return delivered; + } + } + +} diff --git a/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/LMTPProtocolHandlerChain.java b/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/LMTPProtocolHandlerChain.java index 2f0b02c7fe4..2afb104e952 100644 --- a/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/LMTPProtocolHandlerChain.java +++ b/protocols/lmtp/src/test/java/org/apache/james/protocols/lmtp/LMTPProtocolHandlerChain.java @@ -42,6 +42,7 @@ import org.apache.james.protocols.smtp.core.UnknownCmdHandler; import org.apache.james.protocols.smtp.core.VrfyCmdHandler; import org.apache.james.protocols.smtp.core.esmtp.MailSizeEsmtpExtension; +import org.apache.james.protocols.smtp.core.esmtp.SMTPUTF8Extension; import org.apache.james.protocols.smtp.core.esmtp.StartTlsCmdHandler; /** @@ -68,6 +69,7 @@ protected List initDefaultHandlers() { defaultHandlers.add(new VrfyCmdHandler()); defaultHandlers.add(new DataCmdHandler(new RecordingMetricFactory())); defaultHandlers.add(new MailSizeEsmtpExtension()); + defaultHandlers.add(new SMTPUTF8Extension()); defaultHandlers.add(new WelcomeMessageHandler()); defaultHandlers.add(new ReceivedDataLineFilter()); defaultHandlers.add(new DataLineMessageHookHandler()); diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPProtocolHandlerChain.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPProtocolHandlerChain.java index e17fdc847a5..4d1ff29a980 100644 --- a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPProtocolHandlerChain.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPProtocolHandlerChain.java @@ -46,6 +46,7 @@ import org.apache.james.protocols.smtp.core.esmtp.AuthCmdHandler; import org.apache.james.protocols.smtp.core.esmtp.EhloCmdHandler; import org.apache.james.protocols.smtp.core.esmtp.MailSizeEsmtpExtension; +import org.apache.james.protocols.smtp.core.esmtp.SMTPUTF8Extension; import org.apache.james.protocols.smtp.core.esmtp.StartTlsCmdHandler; import org.apache.james.protocols.smtp.hook.Hook; @@ -97,6 +98,7 @@ protected List initDefaultHandlers() { defaultHandlers.add(new VrfyCmdHandler()); defaultHandlers.add(new DataCmdHandler(metricFactory)); defaultHandlers.add(new MailSizeEsmtpExtension()); + defaultHandlers.add(new SMTPUTF8Extension()); defaultHandlers.add(new WelcomeMessageHandler()); defaultHandlers.add(new PostmasterAbuseRcptHook()); defaultHandlers.add(new ReceivedDataLineFilter()); diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPSession.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPSession.java index dfdefa0f7ee..cc6f5dc9f43 100644 --- a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPSession.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/SMTPSession.java @@ -44,6 +44,12 @@ public interface SMTPSession extends ProtocolSession { /** HELO or EHLO */ AttachmentKey CURRENT_HELO_MODE = AttachmentKey.of("CURRENT_HELO_MODE", String.class); AttachmentKey CURRENT_HELO_NAME = AttachmentKey.of("CURRENT_HELO_NAME", String.class); + /** Set per-transaction when the client asserted the RFC 6531 SMTPUTF8 parameter on MAIL FROM. */ + AttachmentKey SMTPUTF8_REQUESTED = AttachmentKey.of("SMTPUTF8_REQUESTED", Boolean.class); + /** The sender address exactly as it arrived on the wire (no bracket removal, no IDN normalisation). Used for echoing back in responses. */ + AttachmentKey RAW_SENDER_STRING = AttachmentKey.of("RAW_SENDER_STRING", String.class); + /** The recipient currently being processed, in wire form. See {@link #RAW_SENDER_STRING}. */ + AttachmentKey RAW_CURRENT_RECIPIENT_STRING = AttachmentKey.of("RAW_CURRENT_RECIPIENT_STRING", String.class); /** * Returns the service wide configuration diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/AddressNormalization.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/AddressNormalization.java new file mode 100644 index 00000000000..3669b361204 --- /dev/null +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/AddressNormalization.java @@ -0,0 +1,76 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.protocols.smtp.core; + +import java.net.IDN; + +import org.apache.james.core.MailAddress; + +/** + * Address-string normalisation helpers shared by MAIL FROM and RCPT TO + * handling. Address validity proper lives in + * {@link org.apache.james.core.MailAddress}; this class is concerned only + * with the protocol-layer transforms. + */ +final class AddressNormalization { + + private AddressNormalization() { + } + + /** + * Convert any {@code xn--} labels (IDNA A-labels) in the domain part of + * {@code address} to their Unicode (U-label) form, leaving the local + * part untouched. Addresses without an {@code @} or without an + * {@code xn--} substring are returned unchanged. + * + * This runs regardless of whether the client declared SMTPUTF8, because + * an A-label-only address is purely ASCII on the wire and has always + * been valid SMTP. Storing the decoded U-label form lets upper layers + * reason about one canonical address. + * + * @throws IllegalArgumentException if any label still starts with + * {@code xn--} after {@link IDN#toUnicode(String, int)} — which + * indicates a malformed A-label that the IDN decoder could not + * interpret. + */ + static String aceLabelsToUnicode(String address) { + int at = address.lastIndexOf('@'); + if (at < 0 || !address.substring(at + 1).contains("xn--")) { + return address; + } + String localPart = address.substring(0, at); + String domain = address.substring(at + 1); + String unicodeDomain = IDN.toUnicode(domain, IDN.ALLOW_UNASSIGNED); + if (unicodeDomain.startsWith("xn--") || unicodeDomain.contains(".xn--")) { + throw new IllegalArgumentException( + "Malformed A-label in domain: " + domain); + } + return localPart + "@" + unicodeDomain; + } + + /** + * Whether {@code s} carries anything outside US-ASCII, i.e. whether the + * client needed to declare SMTPUTF8 (RFC 6531) to send it. Delegates to + * {@link MailAddress#isAscii(String)} so the check has a single definition. + */ + static boolean containsNonAscii(String s) { + return !MailAddress.isAscii(s); + } +} diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/MailCmdHandler.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/MailCmdHandler.java index 1b46315dcaa..972b8a72909 100644 --- a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/MailCmdHandler.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/MailCmdHandler.java @@ -72,6 +72,13 @@ public class MailCmdHandler extends AbstractHookableCmdHandler { DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.ADDRESS_SYNTAX_SENDER) + " Syntax error in sender address").immutable(); + /** RFC 6531 §4.2: 553 5.6.7 when a non-ASCII sender is given without SMTPUTF8. */ + private static final Response NON_ASCII_SENDER_WITHOUT_SMTPUTF8 = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_MAILBOX, + DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.CONTENT_NON_ASCII_ADDR) + + " Non-ASCII addresses not permitted without SMTPUTF8").immutable(); + private static final Response INVALID_IDN_SENDER = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_ARGUMENTS, + DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.ADDRESS_SYNTAX_SENDER) + + " Invalid A-label (xn--) in sender domain").immutable(); /** * A map of parameterHooks */ @@ -104,11 +111,17 @@ public Response onCommand(SMTPSession session, Request request) { private Response doMAIL(SMTPSession session) { StringBuilder responseBuffer = new StringBuilder(); MaybeSender sender = session.getAttachment(SMTPSession.SENDER, State.Transaction).orElse(MaybeSender.nullSender()); + // Echo the sender back in the exact form the client sent it. RFC 6531 + // §3.7.4.2 restricts server responses to ASCII unless SMTPUTF8 is + // asserted, and also lets us preserve the client's choice of + // A-label (xn--) vs U-label when they sent ACE form. + String echo = session.getAttachment(SMTPSession.RAW_SENDER_STRING, State.Transaction) + .orElse(sender.asString()); responseBuffer.append( DSNStatus.getStatus(DSNStatus.SUCCESS, DSNStatus.ADDRESS_OTHER)) .append(" Sender <"); if (!sender.isNullSender()) { - responseBuffer.append(sender.asString()); + responseBuffer.append(echo); } responseBuffer.append("> OK"); @@ -203,8 +216,21 @@ private Response doMAILFilter(SMTPSession session, String argument) { LOGGER.info("Error parsing sender address: {}: did not start and end with < >", sender); return SYNTAX_ERROR; } + String senderAddressString = removeBrackets(sender); + session.setAttachment(SMTPSession.RAW_SENDER_STRING, senderAddressString, State.Transaction); + if (AddressNormalization.containsNonAscii(senderAddressString) + && !session.getAttachment(SMTPSession.SMTPUTF8_REQUESTED, State.Transaction).orElse(Boolean.FALSE)) { + LOGGER.info("Rejected non-ASCII sender address without SMTPUTF8: {}", sender); + return NON_ASCII_SENDER_WITHOUT_SMTPUTF8; + } + try { + senderAddressString = AddressNormalization.aceLabelsToUnicode(senderAddressString); + } catch (IllegalArgumentException e) { + LOGGER.info("Rejected sender address with invalid A-label: {}", sender); + return INVALID_IDN_SENDER; + } try { - MaybeSender senderAddress = toMaybeSender(removeBrackets(sender)); + MaybeSender senderAddress = toMaybeSender(senderAddressString); // Store the senderAddress in session map session.setAttachment(SMTPSession.SENDER, senderAddress, State.Transaction); } catch (Exception pe) { diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/RcptCmdHandler.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/RcptCmdHandler.java index 1ce041ff911..21522b9c212 100644 --- a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/RcptCmdHandler.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/RcptCmdHandler.java @@ -62,6 +62,13 @@ public class RcptCmdHandler extends AbstractHookableCmdHandler impleme private static final Response SYNTAX_ERROR_ARGS = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_ARGUMENTS, DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.DELIVERY_SYNTAX) + " Usage: RCPT TO:").immutable(); private static final Response SYNTAX_ERROR_DELIVERY = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_ARGUMENTS, DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.DELIVERY_SYNTAX) + " Syntax error in parameters or arguments").immutable(); private static final Response SYNTAX_ERROR_ADDRESS = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_MAILBOX, DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.ADDRESS_SYNTAX) + " Syntax error in recipient address").immutable(); + /** RFC 6531 §4.2: 553 5.6.7 when a non-ASCII recipient is given without SMTPUTF8. */ + private static final Response NON_ASCII_RECIPIENT_WITHOUT_SMTPUTF8 = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_MAILBOX, + DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.CONTENT_NON_ASCII_ADDR) + + " Non-ASCII addresses not permitted without SMTPUTF8").immutable(); + private static final Response INVALID_IDN_RECIPIENT = new SMTPResponse(SMTPRetCode.SYNTAX_ERROR_ARGUMENTS, + DSNStatus.getStatus(DSNStatus.PERMANENT, DSNStatus.ADDRESS_SYNTAX) + + " Invalid A-label (xn--) in recipient domain").immutable(); @Inject public RcptCmdHandler(MetricFactory metricFactory) { @@ -87,10 +94,14 @@ protected Response doCoreCmd(SMTPSession session, String command, String paramet rcptColl.add(recipientAddress); session.setAttachment(SMTPSession.RCPT_LIST, rcptColl, State.Transaction); + // Echo the recipient back in the exact form the client sent it. See + // the matching comment in MailCmdHandler.doMAIL — RFC 6531 §3.7.4.2. + String echo = session.getAttachment(SMTPSession.RAW_CURRENT_RECIPIENT_STRING, State.Transaction) + .orElseGet(recipientAddress::asString); StringBuilder response = new StringBuilder(); String status = DSNStatus.getStatus(DSNStatus.SUCCESS, DSNStatus.ADDRESS_VALID); response.append(status) - .append(" Recipient <").append(recipientAddress).append("> OK"); + .append(" Recipient <").append(echo).append("> OK"); LOGGER.debug("RCPT TO {}", StringUtils.abbreviate(recipientAddress.asString(), 80)); @@ -153,6 +164,21 @@ protected Response doFilterChecks(SMTPSession session, String command, + getDefaultDomain(); } + session.setAttachment(SMTPSession.RAW_CURRENT_RECIPIENT_STRING, recipient, State.Transaction); + + if (AddressNormalization.containsNonAscii(recipient) + && !session.getAttachment(SMTPSession.SMTPUTF8_REQUESTED, State.Transaction).orElse(Boolean.FALSE)) { + LOGGER.info("Rejected non-ASCII recipient address without SMTPUTF8: {}", recipient); + return NON_ASCII_RECIPIENT_WITHOUT_SMTPUTF8; + } + + try { + recipient = AddressNormalization.aceLabelsToUnicode(recipient); + } catch (IllegalArgumentException e) { + LOGGER.info("Rejected recipient address with invalid A-label: {}", recipient); + return INVALID_IDN_RECIPIENT; + } + try { recipientAddress = new MailAddress(recipient); } catch (Exception pe) { diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGenerator.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGenerator.java index 614fb5e96ea..4ed4ede28d1 100644 --- a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGenerator.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGenerator.java @@ -40,6 +40,10 @@ public class ReceivedHeaderGenerator { private static final String ESMTPSA = "ESMTPSA"; private static final String ESMTP = "ESMTP"; private static final String ESMTPS = "ESMTPS"; + private static final String UTF8SMTP = "UTF8SMTP"; + private static final String UTF8SMTPA = "UTF8SMTPA"; + private static final String UTF8SMTPS = "UTF8SMTPS"; + private static final String UTF8SMTPSA = "UTF8SMTPSA"; private final ProtocolSession.AttachmentKey mtPriority = ProtocolSession.AttachmentKey.of("MT-PRIORITY", Integer.class); /** @@ -48,24 +52,32 @@ public class ReceivedHeaderGenerator { protected String getServiceType(SMTPSession session, String heloMode) { // Check if EHLO was used if (EHLO.equals(heloMode)) { + // See RFC 6531 §4.3: + // The new keyword "UTF8SMTP" indicates the use of ESMTP when + // the SMTPUTF8 extension is also used; the "A" / "S" / "SA" + // suffixes have the same meaning as in the E* keywords. + boolean smtpUtf8 = session.getAttachment(SMTPSession.SMTPUTF8_REQUESTED, ProtocolSession.State.Transaction) + .orElse(Boolean.FALSE); // Not successful auth if (session.getUsername() == null) { if (session.isTLSStarted()) { - return ESMTPS; + return smtpUtf8 ? UTF8SMTPS : ESMTPS; } else { - return ESMTP; + return smtpUtf8 ? UTF8SMTP : ESMTP; } } else { // See RFC3848: // The new keyword "ESMTPA" indicates the use of ESMTP when the SMTP // AUTH [3] extension is also used and authentication is successfully achieved. if (session.isTLSStarted()) { - return ESMTPSA; + return smtpUtf8 ? UTF8SMTPSA : ESMTPSA; } else { - return ESMTPA; + return smtpUtf8 ? UTF8SMTPA : ESMTPA; } } } else { + // HELO was used (not EHLO), so SMTPUTF8 cannot have been + // negotiated — the extension requires EHLO. Plain SMTP only. return SMTP; } } diff --git a/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/smtputf8/SmtpUtf8MailHook.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/esmtp/SMTPUTF8Extension.java similarity index 61% rename from server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/smtputf8/SmtpUtf8MailHook.java rename to protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/esmtp/SMTPUTF8Extension.java index 1a25b921c00..c54a2219b44 100644 --- a/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/smtputf8/SmtpUtf8MailHook.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/core/esmtp/SMTPUTF8Extension.java @@ -17,32 +17,45 @@ * under the License. * ****************************************************************/ -package org.apache.james.smtpserver.smtputf8; +package org.apache.james.protocols.smtp.core.esmtp; +import java.util.Collections; import java.util.List; +import org.apache.james.protocols.api.ProtocolSession.State; import org.apache.james.protocols.smtp.SMTPSession; -import org.apache.james.protocols.smtp.core.esmtp.EhloExtension; import org.apache.james.protocols.smtp.hook.HookResult; import org.apache.james.protocols.smtp.hook.MailParametersHook; -import org.apache.mailet.Experimental; -import com.google.common.collect.ImmutableList; +/** + * RFC 6531 SMTPUTF8 extension. + * + * Advertises the {@code SMTPUTF8} EHLO keyword and parses the {@code SMTPUTF8} + * parameter on {@code MAIL FROM}. The parameter takes no value; its presence + * on a transaction authorises the use of UTF-8 in the envelope addresses. + * + * Gating of UTF-8 addresses themselves lives in {@code MailCmdHandler} / + * {@code RcptCmdHandler}, which reject non-ASCII addresses with 553 5.6.7 + * when {@link SMTPSession#SMTPUTF8_REQUESTED} is not set. + */ +public class SMTPUTF8Extension implements MailParametersHook, EhloExtension { + + private static final String[] MAIL_PARAMS = { "SMTPUTF8" }; + private static final List FEATURES = Collections.singletonList("SMTPUTF8"); -@Experimental -public class SmtpUtf8MailHook implements MailParametersHook, EhloExtension { @Override public HookResult doMailParameter(SMTPSession session, String paramName, String paramValue) { - return HookResult.DECLINED; + session.setAttachment(SMTPSession.SMTPUTF8_REQUESTED, Boolean.TRUE, State.Transaction); + return null; } @Override public String[] getMailParamNames() { - return new String[]{"SMTPUTF8"}; + return MAIL_PARAMS; } @Override public List getImplementedEsmtpFeatures(SMTPSession session) { - return ImmutableList.of("SMTPUTF8"); + return FEATURES; } } diff --git a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/dsn/DSNStatus.java b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/dsn/DSNStatus.java index 47fc509a44a..55706ef39c8 100644 --- a/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/dsn/DSNStatus.java +++ b/protocols/smtp/src/main/java/org/apache/james/protocols/smtp/dsn/DSNStatus.java @@ -290,6 +290,11 @@ public class DSNStatus { */ public static final String CONTENT_CONVERSION_FAILED = "6.5"; + /** + * Non-ASCII addresses not permitted for that sender/recipient (RFC 6531) + */ + public static final String CONTENT_NON_ASCII_ADDR = "6.7"; + /** * Security or Policy Status diff --git a/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPSServerTest.java b/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPSServerTest.java index 442a3ec87ff..6a5361a0a73 100644 --- a/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPSServerTest.java +++ b/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPSServerTest.java @@ -1,47 +1,63 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.protocols.smtp; - -import org.apache.commons.net.smtp.SMTPClient; -import org.apache.commons.net.smtp.SMTPSClient; -import org.apache.james.protocols.api.Protocol; -import org.apache.james.protocols.api.ProtocolServer; -import org.apache.james.protocols.api.utils.BogusSslContextFactory; -import org.apache.james.protocols.api.utils.BogusTrustManagerFactory; -import org.apache.james.protocols.netty.Encryption; - - -public abstract class AbstractSMTPSServerTest extends AbstractSMTPServerTest { - - - @Override - protected SMTPClient createClient() { - SMTPSClient client = new SMTPSClient(true,BogusSslContextFactory.getClientContext()); - client.setTrustManager(BogusTrustManagerFactory.getTrustManagers()[0]); - return client; - } - - - @Override - protected ProtocolServer createServer(Protocol protocol) { - return createEncryptedServer(protocol, Encryption.createTls(BogusSslContextFactory.getServerContext())); - } - - protected abstract ProtocolServer createEncryptedServer(Protocol protocol, Encryption enc); -} +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.protocols.smtp; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.net.Socket; + +import org.apache.commons.net.smtp.SMTPClient; +import org.apache.commons.net.smtp.SMTPSClient; +import org.apache.james.protocols.api.Protocol; +import org.apache.james.protocols.api.ProtocolServer; +import org.apache.james.protocols.api.utils.BogusSslContextFactory; +import org.apache.james.protocols.api.utils.BogusTrustManagerFactory; +import org.apache.james.protocols.netty.Encryption; + + +public abstract class AbstractSMTPSServerTest extends AbstractSMTPServerTest { + + + @Override + protected SMTPClient createClient() { + SMTPSClient client = new SMTPSClient(true,BogusSslContextFactory.getClientContext()); + client.setTrustManager(BogusTrustManagerFactory.getTrustManagers()[0]); + return client; + } + + + @Override + protected ProtocolServer createServer(Protocol protocol) { + return createEncryptedServer(protocol, Encryption.createTls(BogusSslContextFactory.getServerContext())); + } + + protected abstract ProtocolServer createEncryptedServer(Protocol protocol, Encryption enc); + + /** + * The UTF-8 tests drive a raw socket to control the bytes on the wire; under + * implicit TLS that socket has to speak TLS too, otherwise the exchange + * silently reads back nothing. + */ + @Override + protected Socket createRawSocket(InetSocketAddress address) throws IOException { + return BogusSslContextFactory.getClientContext() + .getSocketFactory() + .createSocket(address.getAddress().getHostAddress(), address.getPort()); + } +} diff --git a/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPServerTest.java b/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPServerTest.java index 71d8979c16d..b412ba02862 100644 --- a/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPServerTest.java +++ b/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/AbstractSMTPServerTest.java @@ -1,988 +1,1314 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ -package org.apache.james.protocols.smtp; - -import static org.assertj.core.api.Assertions.assertThat; - -import java.io.BufferedReader; -import java.io.IOException; -import java.io.InputStreamReader; -import java.net.InetSocketAddress; -import java.net.SocketException; -import java.nio.charset.StandardCharsets; -import java.time.Duration; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; -import java.util.concurrent.atomic.AtomicBoolean; - -import org.apache.commons.net.smtp.SMTPClient; -import org.apache.commons.net.smtp.SMTPReply; -import org.apache.james.core.MailAddress; -import org.apache.james.core.MaybeSender; -import org.apache.james.metrics.tests.RecordingMetricFactory; -import org.apache.james.protocols.api.Protocol; -import org.apache.james.protocols.api.ProtocolServer; -import org.apache.james.protocols.api.handler.ConnectHandler; -import org.apache.james.protocols.api.handler.DisconnectHandler; -import org.apache.james.protocols.api.handler.ProtocolHandler; -import org.apache.james.protocols.api.handler.WiringException; -import org.apache.james.protocols.api.utils.ProtocolServerUtils; -import org.apache.james.protocols.smtp.hook.HeloHook; -import org.apache.james.protocols.smtp.hook.HookResult; -import org.apache.james.protocols.smtp.hook.MailHook; -import org.apache.james.protocols.smtp.hook.MessageHook; -import org.apache.james.protocols.smtp.hook.RcptHook; -import org.apache.james.protocols.smtp.utils.TestMessageHook; -import org.apache.james.util.concurrency.ConcurrentTestRunner; -import org.junit.jupiter.api.Test; - -import com.google.common.io.CharStreams; - -public abstract class AbstractSMTPServerTest { - - protected static final String MSG1 = "Subject: Testmessage\r\n\r\nThis is a message\r\n"; - protected static final String SENDER = "me@sender"; - protected static final String RCPT1 = "rpct1@domain"; - protected static final String RCPT2 = "rpct2@domain"; - - @Test - void testSimpleDelivery() throws Exception { - TestMessageHook hook = new TestMessageHook(); - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - - send(server, bindedAddress, MSG1); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isTrue(); - - MailEnvelope env = queued.next(); - checkEnvelope(env, SENDER, Arrays.asList(RCPT1, RCPT2), MSG1); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - protected void testDeliveryWith4SimultaneousThreads() throws Exception { - TestMessageHook hook = new TestMessageHook(); - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - ProtocolServer finalServer = server; - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - String mailContent = CharStreams.toString(new InputStreamReader(ClassLoader.getSystemResourceAsStream("a50.eml"), StandardCharsets.US_ASCII)); - - ConcurrentTestRunner.builder() - .operation((threadNumber, step) -> send(finalServer, bindedAddress, mailContent)) - .threadCount(4) - .runSuccessfullyWithin(Duration.ofMinutes(1)); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isTrue(); - - MailEnvelope env = queued.next(); - checkEnvelope(env, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); - assertThat(queued.hasNext()).isTrue(); - MailEnvelope env2 = queued.next(); - checkEnvelope(env2, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); - assertThat(queued.hasNext()).isTrue(); - MailEnvelope env3 = queued.next(); - checkEnvelope(env3, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); - assertThat(queued.hasNext()).isTrue(); - MailEnvelope env4 = queued.next(); - checkEnvelope(env4, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - } - - private void send(ProtocolServer server, InetSocketAddress bindedAddress, String msg) throws SocketException, IOException { - SMTPClient client = createClient(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT2); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - assertThat(client.sendShortMessageData(msg)).isTrue(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - } - - @Test - void testStartTlsNotSupported() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.sendCommand("STARTTLS"); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testUnknownCommand() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.sendCommand("UNKNOWN"); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testNoop() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.noop(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - protected void testMailWithoutBrackets() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.mail("invalid"); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - - @Test - void testInvalidHelo() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo(""); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - protected void testRcptWithoutBrackets() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.rcpt(RCPT1); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - protected void testInvalidNoBracketsEnformance() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - Protocol protocol = createProtocol(hook); - ((SMTPConfigurationImpl) protocol.getConfiguration()).setUseAddressBracketsEnforcement(false); - server = createServer(protocol); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.mail(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - protected void testHeloEnforcement() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.setSender(SENDER); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - protected void testHeloEnforcementDisabled() throws Exception { - TestMessageHook hook = new TestMessageHook(); - - ProtocolServer server = null; - try { - Protocol protocol = createProtocol(hook); - ((SMTPConfigurationImpl) protocol.getConfiguration()).setHeloEhloEnforcement(false); - server = createServer(protocol); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = hook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - void testHeloHookPermanentError() throws Exception { - HeloHook hook = (session, helo) -> HookResult.DENY; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - void testHeloHookTempraryError() throws Exception { - HeloHook hook = (session, helo) -> HookResult.DENYSOFT; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testMailHookPermanentError() throws Exception { - MailHook hook = new MailHook() { - @Override - public HookResult doMail(SMTPSession session, MaybeSender sender) { - return HookResult.DENY; - } - }; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testMailHookTemporaryError() throws Exception { - MailHook hook = new MailHook() { - @Override - public HookResult doMail(SMTPSession session, MaybeSender sender) { - return HookResult.DENYSOFT; - } - }; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - void testRcptHookPermanentError() throws Exception { - RcptHook hook = new RcptHook() { - @Override - public HookResult doRcpt(SMTPSession session, MaybeSender sender, MailAddress rcpt) { - if (RCPT1.equals(rcpt.toString())) { - return HookResult.DENY; - } else { - return HookResult.DECLINED; - } - } - - }; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.addRecipient(RCPT2); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - - @Test - void testRcptHookTemporaryError() throws Exception { - RcptHook hook = new RcptHook() { - @Override - public HookResult doRcpt(SMTPSession session, MaybeSender sender, MailAddress rcpt) { - if (RCPT1.equals(rcpt.toString())) { - return HookResult.DENYSOFT; - } else { - return HookResult.DECLINED; - } - } - - }; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.addRecipient(RCPT2); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testNullSender() throws Exception { - ProtocolServer server = null; - try { - server = createServer(createProtocol()); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(""); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT1); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testMessageHookPermanentError() throws Exception { - TestMessageHook testHook = new TestMessageHook(); - - MessageHook hook = (session, mail) -> HookResult.DENY; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook, testHook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT2); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - assertThat(client.sendShortMessageData(MSG1)).isFalse(); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = testHook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - void testMessageHookTemporaryError() throws Exception { - TestMessageHook testHook = new TestMessageHook(); - - MessageHook hook = (session, mail) -> HookResult.DENYSOFT; - - ProtocolServer server = null; - try { - server = createServer(createProtocol(hook, testHook)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - - client.helo("localhost"); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.setSender(SENDER); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.addRecipient(RCPT2); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - assertThat(client.sendShortMessageData(MSG1)).isFalse(); - assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.quit(); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - client.disconnect(); - - Iterator queued = testHook.getQueued().iterator(); - assertThat(queued.hasNext()).isFalse(); - - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - - @Test - void testConnectHandlerPermananet() throws Exception { - ConnectHandler connectHandler = session -> new SMTPResponse("554", "Bye Bye"); - - ProtocolServer server = null; - try { - - server = createServer(createProtocol(connectHandler)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.disconnect(); - - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - - @Test - void testConnectHandlerTemporary() throws Exception { - ConnectHandler connectHandler = session -> new SMTPResponse("451", "Bye Bye"); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(connectHandler)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.disconnect(); - - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - @Test - void testDisconnectHandler() throws Exception { - - final AtomicBoolean called = new AtomicBoolean(false); - DisconnectHandler handler = session -> called.set(true); - - ProtocolServer server = null; - try { - server = createServer(createProtocol(handler)); - server.bind(); - - SMTPClient client = createClient(); - InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); - client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); - assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); - - client.disconnect(); - - Thread.sleep(1000); - assertThat(called.get()).isTrue(); - - - } finally { - if (server != null) { - server.unbind(); - } - } - - } - - protected SMTPClient createClient() { - return new SMTPClient(); - } - - protected abstract ProtocolServer createServer(Protocol protocol); - - - protected Protocol createProtocol(ProtocolHandler... handlers) throws WiringException { - SMTPProtocolHandlerChain chain = new SMTPProtocolHandlerChain(new RecordingMetricFactory()); - chain.addAll(0, Arrays.asList(handlers)); - chain.wireExtensibleHandlers(); - return new SMTPProtocol(chain, new SMTPConfigurationImpl()); - } - - protected static void checkEnvelope(MailEnvelope env, String sender, List recipients, String msg) throws IOException { - assertThat(env.getMaybeSender().asString()).isEqualTo(sender); - - List envRecipients = env.getRecipients(); - assertThat(envRecipients.size()).isEqualTo(recipients.size()); - for (int i = 0; i < recipients.size(); i++) { - MailAddress address = envRecipients.get(i); - assertThat(address.toString()).isEqualTo(recipients.get(i)); - } - - try (BufferedReader reader = new BufferedReader(new InputStreamReader(env.getMessageInputStream()))) { - - String line = null; - boolean start = false; - StringBuilder sb = new StringBuilder(); - while ((line = reader.readLine()) != null) { - if (line.startsWith("Subject")) { - start = true; - } - if (start) { - sb.append(line); - sb.append("\r\n"); - } - } - String msgQueued = sb.subSequence(0, sb.length()).toString(); - - assertThat(msgQueued.length()).isEqualTo(msg.length()); - for (int i = 0; i < msg.length(); i++) { - assertThat(msgQueued.charAt(i)).isEqualTo(msg.charAt(i)); - } - } - - } - -} +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ +package org.apache.james.protocols.smtp; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.InputStreamReader; +import java.net.InetSocketAddress; +import java.net.SocketException; +import java.nio.charset.StandardCharsets; +import java.time.Duration; +import java.util.Arrays; +import java.util.Iterator; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +import org.apache.commons.net.smtp.SMTPClient; +import org.apache.commons.net.smtp.SMTPReply; +import org.apache.james.core.MailAddress; +import org.apache.james.core.MaybeSender; +import org.apache.james.metrics.tests.RecordingMetricFactory; +import org.apache.james.protocols.api.Protocol; +import org.apache.james.protocols.api.ProtocolServer; +import org.apache.james.protocols.api.handler.ConnectHandler; +import org.apache.james.protocols.api.handler.DisconnectHandler; +import org.apache.james.protocols.api.handler.ProtocolHandler; +import org.apache.james.protocols.api.handler.WiringException; +import org.apache.james.protocols.api.utils.ProtocolServerUtils; +import org.apache.james.protocols.smtp.hook.HeloHook; +import org.apache.james.protocols.smtp.hook.HookResult; +import org.apache.james.protocols.smtp.hook.MailHook; +import org.apache.james.protocols.smtp.hook.MessageHook; +import org.apache.james.protocols.smtp.hook.RcptHook; +import org.apache.james.protocols.smtp.utils.TestMessageHook; +import org.apache.james.util.concurrency.ConcurrentTestRunner; +import org.junit.jupiter.api.Test; + +import com.google.common.io.CharStreams; + +public abstract class AbstractSMTPServerTest { + + protected static final String MSG1 = "Subject: Testmessage\r\n\r\nThis is a message\r\n"; + protected static final String SENDER = "me@sender"; + protected static final String RCPT1 = "rpct1@domain"; + protected static final String RCPT2 = "rpct2@domain"; + + @Test + void testSimpleDelivery() throws Exception { + TestMessageHook hook = new TestMessageHook(); + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + send(server, bindedAddress, MSG1); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isTrue(); + + MailEnvelope env = queued.next(); + checkEnvelope(env, SENDER, Arrays.asList(RCPT1, RCPT2), MSG1); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + protected void testDeliveryWith4SimultaneousThreads() throws Exception { + TestMessageHook hook = new TestMessageHook(); + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + ProtocolServer finalServer = server; + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + String mailContent = CharStreams.toString(new InputStreamReader(ClassLoader.getSystemResourceAsStream("a50.eml"), StandardCharsets.US_ASCII)); + + ConcurrentTestRunner.builder() + .operation((threadNumber, step) -> send(finalServer, bindedAddress, mailContent)) + .threadCount(4) + .runSuccessfullyWithin(Duration.ofMinutes(1)); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isTrue(); + + MailEnvelope env = queued.next(); + checkEnvelope(env, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); + assertThat(queued.hasNext()).isTrue(); + MailEnvelope env2 = queued.next(); + checkEnvelope(env2, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); + assertThat(queued.hasNext()).isTrue(); + MailEnvelope env3 = queued.next(); + checkEnvelope(env3, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); + assertThat(queued.hasNext()).isTrue(); + MailEnvelope env4 = queued.next(); + checkEnvelope(env4, SENDER, Arrays.asList(RCPT1, RCPT2), mailContent); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + } + + private void send(ProtocolServer server, InetSocketAddress bindedAddress, String msg) throws SocketException, IOException { + SMTPClient client = createClient(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT2); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + assertThat(client.sendShortMessageData(msg)).isTrue(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + } + + @Test + void testStartTlsNotSupported() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.sendCommand("STARTTLS"); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testUnknownCommand() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.sendCommand("UNKNOWN"); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testNoop() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.noop(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + protected void testMailWithoutBrackets() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.mail("invalid"); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + + @Test + void testInvalidHelo() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo(""); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + protected void testRcptWithoutBrackets() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.rcpt(RCPT1); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + protected void testInvalidNoBracketsEnformance() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + Protocol protocol = createProtocol(hook); + ((SMTPConfigurationImpl) protocol.getConfiguration()).setUseAddressBracketsEnforcement(false); + server = createServer(protocol); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.mail(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + protected void testHeloEnforcement() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.setSender(SENDER); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + protected void testHeloEnforcementDisabled() throws Exception { + TestMessageHook hook = new TestMessageHook(); + + ProtocolServer server = null; + try { + Protocol protocol = createProtocol(hook); + ((SMTPConfigurationImpl) protocol.getConfiguration()).setHeloEhloEnforcement(false); + server = createServer(protocol); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + void testHeloHookPermanentError() throws Exception { + HeloHook hook = (session, helo) -> HookResult.DENY; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + void testHeloHookTempraryError() throws Exception { + HeloHook hook = (session, helo) -> HookResult.DENYSOFT; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testMailHookPermanentError() throws Exception { + MailHook hook = new MailHook() { + @Override + public HookResult doMail(SMTPSession session, MaybeSender sender) { + return HookResult.DENY; + } + }; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testMailHookTemporaryError() throws Exception { + MailHook hook = new MailHook() { + @Override + public HookResult doMail(SMTPSession session, MaybeSender sender) { + return HookResult.DENYSOFT; + } + }; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + void testRcptHookPermanentError() throws Exception { + RcptHook hook = new RcptHook() { + @Override + public HookResult doRcpt(SMTPSession session, MaybeSender sender, MailAddress rcpt) { + if (RCPT1.equals(rcpt.toString())) { + return HookResult.DENY; + } else { + return HookResult.DECLINED; + } + } + + }; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.addRecipient(RCPT2); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + + @Test + void testRcptHookTemporaryError() throws Exception { + RcptHook hook = new RcptHook() { + @Override + public HookResult doRcpt(SMTPSession session, MaybeSender sender, MailAddress rcpt) { + if (RCPT1.equals(rcpt.toString())) { + return HookResult.DENYSOFT; + } else { + return HookResult.DECLINED; + } + } + + }; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.addRecipient(RCPT2); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testNullSender() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol()); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(""); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT1); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testMessageHookPermanentError() throws Exception { + TestMessageHook testHook = new TestMessageHook(); + + MessageHook hook = (session, mail) -> HookResult.DENY; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook, testHook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT2); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + assertThat(client.sendShortMessageData(MSG1)).isFalse(); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = testHook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + void testMessageHookTemporaryError() throws Exception { + TestMessageHook testHook = new TestMessageHook(); + + MessageHook hook = (session, mail) -> HookResult.DENYSOFT; + + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook, testHook)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + + client.helo("localhost"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.setSender(SENDER); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.addRecipient(RCPT2); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + assertThat(client.sendShortMessageData(MSG1)).isFalse(); + assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.quit(); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + client.disconnect(); + + Iterator queued = testHook.getQueued().iterator(); + assertThat(queued.hasNext()).isFalse(); + + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + + @Test + void testConnectHandlerPermananet() throws Exception { + ConnectHandler connectHandler = session -> new SMTPResponse("554", "Bye Bye"); + + ProtocolServer server = null; + try { + + server = createServer(createProtocol(connectHandler)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isNegativePermanent(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.disconnect(); + + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + + @Test + void testConnectHandlerTemporary() throws Exception { + ConnectHandler connectHandler = session -> new SMTPResponse("451", "Bye Bye"); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(connectHandler)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isNegativeTransient(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.disconnect(); + + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + @Test + void testDisconnectHandler() throws Exception { + + final AtomicBoolean called = new AtomicBoolean(false); + DisconnectHandler handler = session -> called.set(true); + + ProtocolServer server = null; + try { + server = createServer(createProtocol(handler)); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())).as("Reply=" + client.getReplyString()).isTrue(); + + client.disconnect(); + + Thread.sleep(1000); + assertThat(called.get()).isTrue(); + + + } finally { + if (server != null) { + server.unbind(); + } + } + + } + + protected SMTPClient createClient() { + return new SMTPClient(); + } + + protected abstract ProtocolServer createServer(Protocol protocol); + + + protected Protocol createProtocol(ProtocolHandler... handlers) throws WiringException { + SMTPProtocolHandlerChain chain = new SMTPProtocolHandlerChain(new RecordingMetricFactory()); + chain.addAll(0, Arrays.asList(handlers)); + chain.wireExtensibleHandlers(); + return new SMTPProtocol(chain, new SMTPConfigurationImpl()); + } + + protected static void checkEnvelope(MailEnvelope env, String sender, List recipients, String msg) throws IOException { + assertThat(env.getMaybeSender().asString()).isEqualTo(sender); + + List envRecipients = env.getRecipients(); + assertThat(envRecipients.size()).isEqualTo(recipients.size()); + for (int i = 0; i < recipients.size(); i++) { + MailAddress address = envRecipients.get(i); + assertThat(address.toString()).isEqualTo(recipients.get(i)); + } + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(env.getMessageInputStream()))) { + + String line = null; + boolean start = false; + StringBuilder sb = new StringBuilder(); + while ((line = reader.readLine()) != null) { + if (line.startsWith("Subject")) { + start = true; + } + if (start) { + sb.append(line); + sb.append("\r\n"); + } + } + String msgQueued = sb.subSequence(0, sb.length()).toString(); + + assertThat(msgQueued.length()).isEqualTo(msg.length()); + for (int i = 0; i < msg.length(); i++) { + assertThat(msgQueued.charAt(i)).isEqualTo(msg.charAt(i)); + } + } + + } + + /** + * The greeting verb of the protocol under test. LMTP forbids EHLO and uses + * LHLO instead, so the RFC 6531 tests below ask rather than assume -- they + * cover both protocols, which is the point: SMTPUTF8 is an ESMTP extension + * that LMTP inherits. + */ + protected String greetingCommand() { + return "EHLO"; + } + + // RFC 6531 SMTPUTF8 + + @Test + void ehloShouldAdvertiseSmtpUtf8() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + client.sendCommand(greetingCommand(), "localhost"); + + assertThat(client.getReplyString()).contains("SMTPUTF8"); + + client.quit(); + client.disconnect(); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void mailFromWithNonAsciiSenderShouldBeRejectedWhenSmtpUtf8NotAsserted() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + client.sendCommand(greetingCommand(), "localhost"); + + client.sendCommand("MAIL", "FROM:"); + + assertThat(client.getReplyCode()).isEqualTo(553); + assertThat(client.getReplyString()).contains("5.6.7"); + + client.quit(); + client.disconnect(); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void mailFromWithNonAsciiSenderShouldBeAcceptedWhenSmtpUtf8IsAsserted() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + String reply = rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM: SMTPUTF8\r\n", + "QUIT\r\n"); + + // RFC 6531 §3.7.4.2: the server echoes the UTF-8 sender address + // back unmodified. + assertThat(reply).contains("250 2.1.0 Sender OK"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void rcptToWithNonAsciiRecipientShouldBeRejectedWhenSmtpUtf8NotAsserted() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + client.sendCommand(greetingCommand(), "localhost"); + client.sendCommand("MAIL", "FROM:<" + SENDER + ">"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())) + .as("Reply=" + client.getReplyString()).isTrue(); + + client.sendCommand("RCPT", "TO:"); + + assertThat(client.getReplyCode()).isEqualTo(553); + assertThat(client.getReplyString()).contains("5.6.7"); + + client.quit(); + client.disconnect(); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void rcptToWithNonAsciiRecipientShouldBeAcceptedWhenSmtpUtf8IsAsserted() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + String reply = rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM:<" + SENDER + "> SMTPUTF8\r\n", + "RCPT TO:\r\n", + "QUIT\r\n"); + + // RFC 6531 §3.7.4.2: the server echoes the UTF-8 recipient address + // back unmodified. + assertThat(reply).contains("250 2.1.5 Recipient OK"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void aceLabelDomainsShouldBeExposedToHooksAsUnicode() throws Exception { + // Drive a full transaction with ACE-form addresses on the wire, then + // inspect the envelope that TestMessageHook captured: both sender and + // recipient should be in U-label form (grå.org), not the ACE form the + // client sent. + TestMessageHook hook = new TestMessageHook(); + ProtocolServer server = null; + try { + server = createServer(createProtocol(hook)); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM:\r\n", + "RCPT TO:\r\n", + "DATA\r\n", + MSG1 + "\r\n.\r\n", + "QUIT\r\n"); + + Iterator queued = hook.getQueued().iterator(); + assertThat(queued.hasNext()).isTrue(); + MailEnvelope env = queued.next(); + assertThat(env.getMaybeSender().asString()).isEqualTo("arnt@grå.org"); + assertThat(env.getRecipients()) + .extracting(MailAddress::asString) + .containsExactly("someone@grå.org"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void mailFromWithAceLabelDomainShouldBeAcceptedWithoutSmtpUtf8() throws Exception { + // xn--gr-zia is the Punycode (A-label) form of "grå". The wire is pure + // ASCII, no SMTPUTF8 asserted. RFC 6531 §3.7.4.2 says the server + // response must stay ASCII in that case, so the echo preserves the + // ACE form the client sent — even though internally we store the + // U-label form (see aceLabelDomainsShouldBeExposedToHooksAsUnicode). + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + String reply = rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM:\r\n", + "QUIT\r\n"); + + assertThat(reply).contains("250 2.1.0 Sender OK"); + assertThat(reply).doesNotContain("grå"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void rcptToWithAceLabelDomainShouldBeAcceptedWithoutSmtpUtf8() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + String reply = rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM:<" + SENDER + ">\r\n", + "RCPT TO:\r\n", + "QUIT\r\n"); + + assertThat(reply).contains("250 2.1.5 Recipient OK"); + assertThat(reply).doesNotContain("grå"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void mailFromWithMalformedAceLabelShouldBeRejected() throws Exception { + // "xn--" on its own is not a valid A-label; IDN.toUnicode leaves it + // unchanged, which we detect and reject with a specific error. + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + String reply = rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM:\r\n", + "QUIT\r\n"); + + assertThat(reply).contains("501"); + assertThat(reply).contains("Invalid A-label"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + @Test + void rcptToWithMalformedAceLabelShouldBeRejected() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + + String reply = rawUtf8Exchange(bindedAddress, + greetingCommand() + " localhost\r\n", + "MAIL FROM:<" + SENDER + ">\r\n", + "RCPT TO:\r\n", + "QUIT\r\n"); + + assertThat(reply).contains("501"); + assertThat(reply).contains("Invalid A-label"); + } finally { + if (server != null) { + server.unbind(); + } + } + } + + /** + * Write all {@code commands} verbatim in UTF-8 and return the concatenated + * server response as one UTF-8 decoded string. Reads until the server + * closes the socket (which it does on QUIT). + */ + /** + * Opens the socket {@link #rawUtf8Exchange} talks over. Overridden by the + * implicit-TLS variants so the raw exchanges below run there too, rather + * than being skipped for lack of an SSL-aware helper. + */ + protected java.net.Socket createRawSocket(InetSocketAddress address) throws IOException { + return new java.net.Socket(address.getAddress().getHostAddress(), address.getPort()); + } + + private String rawUtf8Exchange(InetSocketAddress address, String... commands) throws IOException { + try (java.net.Socket socket = createRawSocket(address)) { + socket.getOutputStream().write(String.join("", commands).getBytes(StandardCharsets.UTF_8)); + socket.getOutputStream().flush(); + java.io.ByteArrayOutputStream collected = new java.io.ByteArrayOutputStream(); + byte[] buf = new byte[4096]; + int n; + while ((n = socket.getInputStream().read(buf)) > 0) { + collected.write(buf, 0, n); + } + return collected.toString(StandardCharsets.UTF_8); + } + } + + @Test + void asciiAddressesShouldStillWorkWithoutSmtpUtf8() throws Exception { + ProtocolServer server = null; + try { + server = createServer(createProtocol(new TestMessageHook())); + server.bind(); + + SMTPClient client = createClient(); + InetSocketAddress bindedAddress = new ProtocolServerUtils(server).retrieveBindedAddress(); + client.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + client.sendCommand(greetingCommand(), "localhost"); + + client.sendCommand("MAIL", "FROM:"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())) + .as("Reply=" + client.getReplyString()).isTrue(); + + client.sendCommand("RCPT", "TO:"); + assertThat(SMTPReply.isPositiveCompletion(client.getReplyCode())) + .as("Reply=" + client.getReplyString()).isTrue(); + + client.quit(); + client.disconnect(); + } finally { + if (server != null) { + server.unbind(); + } + } + } + +} diff --git a/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGeneratorTest.java b/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGeneratorTest.java new file mode 100644 index 00000000000..070b66ba53c --- /dev/null +++ b/protocols/smtp/src/test/java/org/apache/james/protocols/smtp/core/ReceivedHeaderGeneratorTest.java @@ -0,0 +1,141 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.protocols.smtp.core; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.Optional; + +import org.apache.james.core.Username; +import org.apache.james.protocols.api.ProtocolSession.AttachmentKey; +import org.apache.james.protocols.api.ProtocolSession.State; +import org.apache.james.protocols.smtp.SMTPSession; +import org.apache.james.protocols.smtp.utils.BaseFakeSMTPSession; +import org.junit.jupiter.api.Test; + +/** + * Covers RFC 6531 §4.3 — the UTF8SMTP / UTF8SMTPA / UTF8SMTPS / UTF8SMTPSA + * trace keywords used in the Received header when the transaction asserted + * SMTPUTF8. + */ +class ReceivedHeaderGeneratorTest { + + private static final ReceivedHeaderGenerator generator = new ReceivedHeaderGenerator(); + + private static String serviceTypeFor(String heloMode, boolean tls, boolean authenticated, boolean smtpUtf8) { + SMTPSession session = new FakeSession(tls, authenticated, smtpUtf8); + // getServiceType is protected; we exercise it through a thin + // subclass that exposes it. + return new ReceivedHeaderGenerator() { + String invoke() { + return getServiceType(session, heloMode); + } + }.invoke(); + } + + // --- HELO (no extensions can have been negotiated) --- + + @Test + void heloShouldYieldSmtp() { + assertThat(serviceTypeFor("HELO", false, false, false)).isEqualTo("SMTP"); + } + + @Test + void heloShouldYieldSmtpEvenWhenSmtpUtf8FlagIsSet() { + // The flag should only ever be set after EHLO + an SMTPUTF8 + // parameter, but we double-check the HELO branch ignores it. + assertThat(serviceTypeFor("HELO", false, false, true)).isEqualTo("SMTP"); + } + + // --- EHLO without SMTPUTF8: existing RFC 3848 keywords --- + + @Test + void ehloShouldYieldEsmtp() { + assertThat(serviceTypeFor("EHLO", false, false, false)).isEqualTo("ESMTP"); + } + + @Test + void ehloAuthenticatedShouldYieldEsmtpa() { + assertThat(serviceTypeFor("EHLO", false, true, false)).isEqualTo("ESMTPA"); + } + + @Test + void ehloOverTlsShouldYieldEsmtps() { + assertThat(serviceTypeFor("EHLO", true, false, false)).isEqualTo("ESMTPS"); + } + + @Test + void ehloOverTlsAuthenticatedShouldYieldEsmtpsa() { + assertThat(serviceTypeFor("EHLO", true, true, false)).isEqualTo("ESMTPSA"); + } + + // --- EHLO with SMTPUTF8: RFC 6531 §4.3 keywords --- + + @Test + void ehloWithSmtpUtf8ShouldYieldUtf8Smtp() { + assertThat(serviceTypeFor("EHLO", false, false, true)).isEqualTo("UTF8SMTP"); + } + + @Test + void ehloAuthenticatedWithSmtpUtf8ShouldYieldUtf8Smtpa() { + assertThat(serviceTypeFor("EHLO", false, true, true)).isEqualTo("UTF8SMTPA"); + } + + @Test + void ehloOverTlsWithSmtpUtf8ShouldYieldUtf8Smtps() { + assertThat(serviceTypeFor("EHLO", true, false, true)).isEqualTo("UTF8SMTPS"); + } + + @Test + void ehloOverTlsAuthenticatedWithSmtpUtf8ShouldYieldUtf8Smtpsa() { + assertThat(serviceTypeFor("EHLO", true, true, true)).isEqualTo("UTF8SMTPSA"); + } + + private static class FakeSession extends BaseFakeSMTPSession { + private final boolean tls; + private final Username username; + private final boolean smtpUtf8; + + FakeSession(boolean tls, boolean authenticated, boolean smtpUtf8) { + this.tls = tls; + this.username = authenticated ? Username.of("alice@example.com") : null; + this.smtpUtf8 = smtpUtf8; + } + + @Override + public boolean isTLSStarted() { + return tls; + } + + @Override + public Username getUsername() { + return username; + } + + @Override + @SuppressWarnings("unchecked") + public Optional getAttachment(AttachmentKey key, State state) { + if (key == SMTPSession.SMTPUTF8_REQUESTED) { + return (Optional) Optional.of(smtpUtf8); + } + return Optional.empty(); + } + } +} diff --git a/server/apps/memory-app/src/test/java/org/apache/james/CertificateReloadTest.java b/server/apps/memory-app/src/test/java/org/apache/james/CertificateReloadTest.java index 7b084f45d74..80bdd10d002 100644 --- a/server/apps/memory-app/src/test/java/org/apache/james/CertificateReloadTest.java +++ b/server/apps/memory-app/src/test/java/org/apache/james/CertificateReloadTest.java @@ -206,7 +206,7 @@ void reloadShouldNotAbortExistingConnections() throws Exception { readBytes(channel); channel.getOutputStream().write("EHLO toto.com\r\n".getBytes(StandardCharsets.UTF_8)); assertThat(readBytes(channel)) - .contains("250 8BITMIME"); + .contains("8BITMIME"); } private String readBytes(SSLSocket sslSocket) throws IOException { diff --git a/server/container/core/src/main/java/org/apache/james/server/core/InternetHeadersInputStream.java b/server/container/core/src/main/java/org/apache/james/server/core/InternetHeadersInputStream.java index 290bc914ebb..7759e89f136 100644 --- a/server/container/core/src/main/java/org/apache/james/server/core/InternetHeadersInputStream.java +++ b/server/container/core/src/main/java/org/apache/james/server/core/InternetHeadersInputStream.java @@ -69,7 +69,7 @@ private boolean readNextLine() { if (!headerLines.hasMoreElements()) { line += LINE_SEPERATOR; } - currLine = line.getBytes(StandardCharsets.US_ASCII); + currLine = line.getBytes(StandardCharsets.UTF_8); return true; } else { return false; diff --git a/server/container/core/src/main/java/org/apache/james/server/core/MailHeaders.java b/server/container/core/src/main/java/org/apache/james/server/core/MailHeaders.java index 71d70e60dcb..be77e707eeb 100644 --- a/server/container/core/src/main/java/org/apache/james/server/core/MailHeaders.java +++ b/server/container/core/src/main/java/org/apache/james/server/core/MailHeaders.java @@ -39,8 +39,8 @@ * */ public class MailHeaders extends InternetHeaders implements Serializable, Cloneable { - private static final long serialVersionUID = 238748126601L; + private static final boolean ALLOWUTF_8 = true; private boolean modified = false; private long size = -1; @@ -67,7 +67,7 @@ public MailHeaders() { */ public MailHeaders(InputStream in) throws MessagingException { super(); - load(in); + load(in, ALLOWUTF_8); } /** diff --git a/server/container/core/src/test/java/org/apache/james/server/core/MimeMessageWrapperTest.java b/server/container/core/src/test/java/org/apache/james/server/core/MimeMessageWrapperTest.java index 7bba53b35d8..a97a03f02a1 100644 --- a/server/container/core/src/test/java/org/apache/james/server/core/MimeMessageWrapperTest.java +++ b/server/container/core/src/test/java/org/apache/james/server/core/MimeMessageWrapperTest.java @@ -26,6 +26,7 @@ import java.io.IOException; import java.io.InputStream; import java.io.InputStreamReader; +import java.nio.charset.StandardCharsets; import java.util.Enumeration; import java.util.Properties; @@ -87,6 +88,7 @@ public synchronized void loadMessage() throws MessagingException { TestableMimeMessageWrapper mw = null; TestableMimeMessageWrapper onlyHeader = null; final String content = "Subject: foo\r\nContent-Transfer-Encoding2: plain"; + final String contentUtf8 = "Subject: fée\r\nContent-Transfer-Encoding2: plain"; final String sep = "\r\n\r\n"; final String body = "bar\r\n"; @@ -276,6 +278,35 @@ public void testSize() throws MessagingException { assertThat(mw.getSize()).isEqualTo(body.length()); } + @Test + public void testSizeUtf8() throws Exception { + TestableMimeMessageWrapper message = getMessageFromSources(contentUtf8 + sep + body); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + message.writeTo(baos); + + assertThat(message.getMessageSize()) + .isEqualTo(baos.size()); + } + + @Test + public void testWriteToUtf8() throws Exception { + TestableMimeMessageWrapper message = getMessageFromSources(contentUtf8 + sep + body); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + message.writeTo(baos); + + assertThat(baos.toString(StandardCharsets.UTF_8)).isEqualTo(contentUtf8 + sep + body); + } + + @Test + public void testWriteToUtf8AfterHeaderModification() throws Exception { + TestableMimeMessageWrapper message = getMessageFromSources(contentUtf8 + sep + body); + message.addHeader("Another", "header"); + ByteArrayOutputStream baos = new ByteArrayOutputStream(); + message.writeTo(baos); + + assertThat(baos.toString(StandardCharsets.UTF_8)).contains("Subject: fée\r\n"); + } + @Test public void getSizeShouldReturnZeroWhenNoHeaderAndAddHeader() throws MessagingException { onlyHeader.addHeader("a", "b"); diff --git a/server/mailet/integration-testing/src/test/java/org/apache/james/mailets/AddDeliveredToHeaderTest.java b/server/mailet/integration-testing/src/test/java/org/apache/james/mailets/AddDeliveredToHeaderTest.java index d5bb6c788aa..17697f813ed 100644 --- a/server/mailet/integration-testing/src/test/java/org/apache/james/mailets/AddDeliveredToHeaderTest.java +++ b/server/mailet/integration-testing/src/test/java/org/apache/james/mailets/AddDeliveredToHeaderTest.java @@ -43,6 +43,9 @@ import org.junit.jupiter.api.io.TempDir; class AddDeliveredToHeaderTest { + private static final String RECIPIENT2_UTF8 = "rené@" + DEFAULT_DOMAIN; + private static final String RECIPIENT2 = "rene@" + DEFAULT_DOMAIN; + @RegisterExtension public TestIMAPClient testIMAPClient = new TestIMAPClient(); @RegisterExtension @@ -58,6 +61,7 @@ void setup(@TempDir File temporaryFolder) throws Exception { DataProbe dataProbe = jamesServer.getProbe(DataProbeImpl.class); dataProbe.addDomain(DEFAULT_DOMAIN); dataProbe.addUser(RECIPIENT, PASSWORD); + dataProbe.addUser(RECIPIENT2, PASSWORD); dataProbe.addUser(FROM, PASSWORD); } @@ -79,4 +83,32 @@ void receivedMessagesShouldContainDeliveredToHeaders() throws Exception { assertThat(testIMAPClient.readFirstMessageHeaders()) .contains(AddDeliveredToHeader.DELIVERED_TO + ": " + RECIPIENT); } + + @Test + void receivedMessagesShouldContainDeliveredToHeadersI8N() throws Exception { + jamesServer.getProbe(DataProbeImpl.class).addUserAliasMapping("rené", DEFAULT_DOMAIN, RECIPIENT2); + String message = "FROM: " + RECIPIENT2_UTF8 + "\r\n" + + "subject: testé\r\n" + + "Content-Type: text/plain; charset=UTF-8\r\n" + + "Content-Encoding: 8bit\r\n" + + "\r\n" + + "contenté\r\n"; + + // A non-ASCII recipient requires the SMTPUTF8 extension (RFC 6531): the transaction + // needs to be opened with EHLO and MAIL FROM ... SMTPUTF8, otherwise RCPT is rejected + // with 553 5.6.7. + messageSender.connect(LOCALHOST_IP, jamesServer.getProbe(SmtpGuiceProbe.class).getSmtpPort()) + .authenticate(FROM, PASSWORD) + .sendMessageWithHeadersSmtpUtf8(FROM, RECIPIENT2_UTF8, message); + + testIMAPClient.connect(LOCALHOST_IP, jamesServer.getProbe(ImapGuiceProbe.class).getImapPort()) + .login(RECIPIENT2, PASSWORD) + .select(TestIMAPClient.INBOX) + .awaitMessage(awaitAtMostOneMinute); + + assertThat(testIMAPClient.readFirstMessage()) + .contains(RECIPIENT2_UTF8) + .contains("testé") + .contains("contenté"); + } } diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/DSNBounce.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/DSNBounce.java index 89dc688e949..c745e3dc2e7 100755 --- a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/DSNBounce.java +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/DSNBounce.java @@ -476,6 +476,7 @@ private String bounceMessage() { private MimeBodyPart createDSN(Mail originalMail) throws MessagingException { StringBuilder buffer = new StringBuilder(); + boolean anyNonAsciiAddress = false; appendReportingMTA(buffer); buffer.append("Received-From-MTA: dns; " + originalMail.getRemoteHost()) @@ -496,12 +497,22 @@ private MimeBodyPart createDSN(Mail originalMail) throws MessagingException { .append(LINE_BREAK)); for (MailAddress rec : originalMail.getRecipients()) { + anyNonAsciiAddress |= !rec.isAscii(); appendRecipient(buffer, rec, getDeliveryError(originalMail), originalMail.getLastUpdated()); } MimeBodyPart bodyPart = new MimeBodyPart(); - bodyPart.setContent(buffer.toString(), "text/plain"); - bodyPart.setHeader("Content-Type", "message/delivery-status"); + // RFC 6533 §3.2: when any reported address contains non-ASCII + // octets the DSN body part is "message/global-delivery-status"; + // otherwise the RFC 3464 form. The outer "multipart/report; + // report-type=delivery-status" wrapper does not change. Setting + // the storage Content-Type with charset=UTF-8 first makes + // jakarta.mail serialise the body as UTF-8 octets; the second + // setHeader overrides only the type label. + bodyPart.setContent(buffer.toString(), "text/plain; charset=UTF-8"); + bodyPart.setHeader("Content-Type", anyNonAsciiAddress + ? "message/global-delivery-status; charset=UTF-8" + : "message/delivery-status"); bodyPart.setDescription("Delivery Status Notification"); bodyPart.setFileName("status.dat"); return bodyPart; @@ -518,7 +529,10 @@ private void appendReportingMTA(StringBuilder buffer) { private void appendRecipient(StringBuilder buffer, MailAddress mailAddress, String deliveryError, Date lastUpdated) { buffer.append(LINE_BREAK); - buffer.append("Final-Recipient: rfc822; " + mailAddress.toString()).append(LINE_BREAK); + // RFC 6533 §3.2: addr-type is "utf-8" when the address contains + // non-ASCII octets, otherwise the legacy "rfc822". + buffer.append("Final-Recipient: ").append(addrType(mailAddress)).append("; ") + .append(mailAddress.toString()).append(LINE_BREAK); buffer.append("Action: ").append(action.asString().toLowerCase(Locale.US)).append(LINE_BREAK); buffer.append("Status: " + deliveryError).append(LINE_BREAK); if (action.shouldIncludeDiagnostic()) { @@ -528,6 +542,10 @@ private void appendRecipient(StringBuilder buffer, MailAddress mailAddress, Stri .append(LINE_BREAK); } + private static String addrType(MailAddress mailAddress) { + return mailAddress.isAscii() ? "rfc822" : "utf-8"; + } + private String getDeliveryError(Mail originalMail) { return AttributeUtils .getValueAndCastFromMail(originalMail, DELIVERY_ERROR, String.class) diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/RemoteDelivery.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/RemoteDelivery.java index c0bf0ab4b89..03698b25b35 100644 --- a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/RemoteDelivery.java +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/RemoteDelivery.java @@ -152,6 +152,21 @@ * or use the mail.smtps.ssl.checkserveridentity and mail.smtp.ssl.checkserveridentity javax properties for fine control.
* Read org.eclipse.angus.mail.smtp * for full information. + *
+ * SMTPUTF8 (RFC 6531): when the first MX we reach advertises + * SMTPUTF8 and the envelope contains any non-ASCII character, the + * extension is asserted on MAIL FROM and UTF-8 addresses flow through + * unchanged. When the MX lacks SMTPUTF8 and only the domain parts are + * non-ASCII, those domains are converted to their ACE (A-label, xn--) + * form for the envelope commands; the message headers themselves are + * still allowed to carry UTF-8 (Subject, display names, etc.). This can + * produce a transaction where RCPT TO carries punycode but the mail + * headers carry UTF-8 — that mismatch is a fair compromise that + * optimises handling on the receiver side: a receiver that cannot speak + * SMTPUTF8 still accepts the envelope it understands, and a receiver + * that can read UTF-8 headers (most do) gets them intact. When a local + * part is non-ASCII and the MX lacks SMTPUTF8 the transaction fails + * permanently — no lossless downgrade exists. */ public class RemoteDelivery extends GenericMailet { private static final Logger LOGGER = LoggerFactory.getLogger(RemoteDelivery.class); diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/MailDelivrerToHost.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/MailDelivrerToHost.java index 75efb77f755..bf47977701e 100644 --- a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/MailDelivrerToHost.java +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/MailDelivrerToHost.java @@ -68,6 +68,7 @@ public class MailDelivrerToHost { public static final String BIT_MIME_8 = "8BITMIME"; public static final String REQUIRE_TLS = "REQUIRETLS"; public static final String STARTTLS = "STARTTLS"; + public static final String SMTPUTF8 = "SMTPUTF8"; public static final String MT_PRIORITY = "MT-PRIORITY"; public static final String MAIL_PRIORITY_ATTRIBUTE_NAME = "MAIL_PRIORITY"; private static final List supportedSmtpExtensionsList = List.of(MT_PRIORITY, STARTTLS); @@ -126,6 +127,20 @@ public ExecutionResult tryDeliveryToHost(Mail mail, Collection // "mail.smtp.dsn.ret" //default to nothing... appended as RET= after MAIL FROM line. // "mail.smtp.dsn.notify" //default to nothing... appended as NOTIFY= after RCPT TO line. + // Angus reads mail.mime.allowutf8 in the SMTPTransport constructor, so it + // has to be decided here -- before the transport exists, hence before we + // can know whether the remote advertises SMTPUTF8. The envelope alone + // tells us whether UTF-8 could ever be needed, and that is enough: Angus + // only emits the SMTPUTF8 keyword on MAIL FROM when the remote also + // advertises it, and SmtpUtf8Strategy below still decides the downgrade. + // Left alone for the ASCII envelopes that make up the bulk of the + // traffic, so they neither change behaviour nor trip Angus' "allowutf8 + // set but server doesn't advertise SMTPUTF8" log line. The pool clears + // the property again when the session is passivated. + if (SmtpUtf8Strategy.envelopeNeedsUtf8(mail.getMaybeSender(), addr)) { + props.put("mail.mime.allowutf8", "true"); + } + SMTPTransport transport = null; try { transport = (SMTPTransport) session.getTransport(outgoingMailServer); @@ -134,6 +149,22 @@ public ExecutionResult tryDeliveryToHost(Mail mail, Collection if (receiverDoesNotProvideNecessaryStartTls(mail, transport)) { return ExecutionResult.permanentFailure(new SendFailedException("Mail delivery failed; the receiving server does not support STARTTLS")); } + // We assume all MXes for a given destination domain advertise + // the same set of SMTP extensions; this decision is made once, + // on the first MX we reach, and we do not fall back to another + // MX hoping it might have different capabilities. + SmtpUtf8Strategy.Action utf8Action = SmtpUtf8Strategy.pick( + mail.getMaybeSender(), addr, transport.supportsExtension(SMTPUTF8)); + if (utf8Action == SmtpUtf8Strategy.Action.CANNOT_DOWNGRADE) { + return ExecutionResult.permanentFailure(new SendFailedException( + "Remote server does not advertise SMTPUTF8 but the envelope " + + "contains a non-ASCII local part that cannot be downgraded")); + } + if (utf8Action == SmtpUtf8Strategy.Action.DOWNGRADE_DOMAINS) { + addr = toAceDomains(addr); + props.put(inContext(session, "mail.smtp.from"), + SmtpUtf8Strategy.aceAddressString(mail.getMaybeSender().asString())); + } if (mail.dsnParameters().isPresent()) { sendDSNAwareEmail(mail, transport, addr); } else if (extensionsSupported(transport)) { @@ -236,6 +267,14 @@ private static boolean extensionsSupported(SMTPTransport transport) { return supportedSmtpExtensionsList.stream().anyMatch(transport::supportsExtension); } + private static Collection toAceDomains(Collection addr) throws MessagingException { + Collection out = new java.util.ArrayList<>(addr.size()); + for (InternetAddress a : addr) { + out.add(SmtpUtf8Strategy.toAceDomain(a)); + } + return out; + } + private static boolean receiverDoesNotProvideNecessaryStartTls(Mail mail, SMTPTransport transport) { return !transport.getLastServerResponse().contains(STARTTLS) && mail.attributesMap().containsKey(AttributeName.of(REQUIRE_TLS)) && diff --git a/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/SmtpUtf8Strategy.java b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/SmtpUtf8Strategy.java new file mode 100644 index 00000000000..629576870a8 --- /dev/null +++ b/server/mailet/mailets/src/main/java/org/apache/james/transport/mailets/remote/delivery/SmtpUtf8Strategy.java @@ -0,0 +1,151 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.mailets.remote.delivery; + +import java.net.IDN; +import java.util.Collection; + +import jakarta.mail.internet.AddressException; +import jakarta.mail.internet.InternetAddress; + +import org.apache.james.core.MailAddress; +import org.apache.james.core.MaybeSender; + +/** + * Picks a relaying strategy for the RFC 6531 SMTPUTF8 extension based on + * what the remote MX advertises and what's in the envelope. We assume all + * MXes for a single destination domain advertise the same extensions; if + * the first MX we try lacks SMTPUTF8 we don't retry subsequent ones hoping + * they'll differ. + */ +public final class SmtpUtf8Strategy { + + public enum Action { + /** No envelope address has non-ASCII characters; deliver as-is. */ + NO_UTF8_NEEDED, + /** Some envelope address has non-ASCII characters and the remote + * advertises SMTPUTF8; deliver as-is and assert SMTPUTF8. */ + USE_EXTENSION, + /** Remote lacks SMTPUTF8 but all non-ASCII lives in the domain + * part, which can be downgraded to ACE (A-label, xn--) form. */ + DOWNGRADE_DOMAINS, + /** Remote lacks SMTPUTF8 and at least one local part is non-ASCII, + * so no lossless downgrade exists. Caller should fail the + * transaction the same way it fails a SIZE overflow. */ + CANNOT_DOWNGRADE + } + + private SmtpUtf8Strategy() { + } + + /** + * Whether any envelope address carries non-ASCII, i.e. whether this + * delivery may need SMTPUTF8 at all. Unlike {@link #pick}, this needs no + * connection to the remote, so the caller can decide before building the + * transport -- Angus reads {@code mail.mime.allowutf8} in the + * {@code SMTPTransport} constructor, too early for {@link #pick}'s verdict. + */ + public static boolean envelopeNeedsUtf8(MaybeSender sender, Collection recipients) { + return hasNonAsciiLocalPart(sender, recipients) || hasNonAsciiDomain(sender, recipients); + } + + public static Action pick(MaybeSender sender, + Collection recipients, + boolean remoteSupportsSmtpUtf8) { + boolean nonAsciiLocalPart = hasNonAsciiLocalPart(sender, recipients); + boolean nonAsciiDomain = hasNonAsciiDomain(sender, recipients); + + if (!nonAsciiLocalPart && !nonAsciiDomain) { + return Action.NO_UTF8_NEEDED; + } + if (remoteSupportsSmtpUtf8) { + return Action.USE_EXTENSION; + } + if (nonAsciiLocalPart) { + return Action.CANNOT_DOWNGRADE; + } + return Action.DOWNGRADE_DOMAINS; + } + + /** + * Returns a copy of {@code address} with its domain converted to ACE + * (A-label) form via {@link IDN#toASCII}. Passing an already-ASCII + * domain through this is a no-op, so callers don't need to check. + * + * @throws AddressException if the address has no {@code @} + */ + public static InternetAddress toAceDomain(InternetAddress address) throws AddressException { + String asString = address.getAddress(); + int at = asString.lastIndexOf('@'); + if (at < 0) { + throw new AddressException("Address has no @: " + asString); + } + String localPart = asString.substring(0, at); + String domain = asString.substring(at + 1); + // InternetAddress(String) parses strictly; bypass via setAddress so + // we don't reject local parts we're only passing through unchanged. + InternetAddress result = new InternetAddress(); + result.setAddress(localPart + "@" + IDN.toASCII(domain, IDN.ALLOW_UNASSIGNED)); + return result; + } + + /** ACE form of the string address. See {@link #toAceDomain}. */ + public static String aceAddressString(String address) { + int at = address.lastIndexOf('@'); + if (at < 0) { + return address; + } + return address.substring(0, at + 1) + + IDN.toASCII(address.substring(at + 1), IDN.ALLOW_UNASSIGNED); + } + + private static boolean hasNonAsciiLocalPart(MaybeSender sender, + Collection recipients) { + if (!sender.isNullSender() + && !MailAddress.isAscii(sender.asString().substring(0, Math.max(0, sender.asString().lastIndexOf('@'))))) { + return true; + } + for (InternetAddress a : recipients) { + int at = a.getAddress().lastIndexOf('@'); + String localPart = at < 0 ? a.getAddress() : a.getAddress().substring(0, at); + if (!MailAddress.isAscii(localPart)) { + return true; + } + } + return false; + } + + private static boolean hasNonAsciiDomain(MaybeSender sender, + Collection recipients) { + if (!sender.isNullSender()) { + int at = sender.asString().lastIndexOf('@'); + if (at >= 0 && !MailAddress.isAscii(sender.asString().substring(at + 1))) { + return true; + } + } + for (InternetAddress a : recipients) { + int at = a.getAddress().lastIndexOf('@'); + if (at >= 0 && !MailAddress.isAscii(a.getAddress().substring(at + 1))) { + return true; + } + } + return false; + } +} diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/DSNBounceTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/DSNBounceTest.java index 80ede282035..8dee8c9d130 100644 --- a/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/DSNBounceTest.java +++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/DSNBounceTest.java @@ -1496,4 +1496,88 @@ void shouldAddAutoSubmittedHeader() throws Exception { assertThat(MimeMessageUtil.asString(sentMessage)) .contains("Auto-Submitted: auto-replied"); } + + @Nested + class Rfc6533 { + @Test + void dsnForAsciiRecipientShouldUseRfc822AddrTypeAndLegacyContentType() throws Exception { + FakeMailetConfig mailetConfig = FakeMailetConfig.builder() + .mailetName(MAILET_NAME) + .mailetContext(fakeMailContext) + .build(); + dsnBounce.init(mailetConfig); + + FakeMail mail = FakeMail.builder() + .name(MAILET_NAME) + .sender(new MailAddress("sender@example.com")) + .attribute(DELIVERY_ERROR_ATTRIBUTE) + .mimeMessage(MimeMessageBuilder.mimeMessageBuilder().setText("body")) + .recipient("info@example.com") + .lastUpdated(Date.from(Instant.parse("2026-04-27T10:00:00.000Z"))) + .remoteAddr("remoteHost") + .build(); + + dsnBounce.service(mail); + + BodyPart dsnPart = (BodyPart) ((MimeMultipart) fakeMailContext.getSentMails() + .get(0).getMsg().getContent()).getBodyPart(1); + assertThat(dsnPart.getContentType()).startsWith("message/delivery-status"); + String body = IOUtils.toString((SharedByteArrayInputStream) dsnPart.getContent(), StandardCharsets.UTF_8); + assertThat(body).contains("Final-Recipient: rfc822; info@example.com"); + } + + @Test + void dsnForUtf8LocalPartShouldUseUtf8AddrTypeAndGlobalContentType() throws Exception { + FakeMailetConfig mailetConfig = FakeMailetConfig.builder() + .mailetName(MAILET_NAME) + .mailetContext(fakeMailContext) + .build(); + dsnBounce.init(mailetConfig); + + FakeMail mail = FakeMail.builder() + .name(MAILET_NAME) + .sender(new MailAddress("sender@example.com")) + .attribute(DELIVERY_ERROR_ATTRIBUTE) + .mimeMessage(MimeMessageBuilder.mimeMessageBuilder().setText("body")) + .recipient("grå@example.com") + .lastUpdated(Date.from(Instant.parse("2026-04-27T10:00:00.000Z"))) + .remoteAddr("remoteHost") + .build(); + + dsnBounce.service(mail); + + BodyPart dsnPart = (BodyPart) ((MimeMultipart) fakeMailContext.getSentMails() + .get(0).getMsg().getContent()).getBodyPart(1); + assertThat(dsnPart.getContentType()).startsWith("message/global-delivery-status"); + String body = IOUtils.toString((SharedByteArrayInputStream) dsnPart.getContent(), StandardCharsets.UTF_8); + assertThat(body).contains("Final-Recipient: utf-8; grå@example.com"); + } + + @Test + void dsnForUtf8DomainOnlyShouldUseUtf8AddrTypeAndGlobalContentType() throws Exception { + FakeMailetConfig mailetConfig = FakeMailetConfig.builder() + .mailetName(MAILET_NAME) + .mailetContext(fakeMailContext) + .build(); + dsnBounce.init(mailetConfig); + + FakeMail mail = FakeMail.builder() + .name(MAILET_NAME) + .sender(new MailAddress("sender@example.com")) + .attribute(DELIVERY_ERROR_ATTRIBUTE) + .mimeMessage(MimeMessageBuilder.mimeMessageBuilder().setText("body")) + .recipient("arnt@grå.org") + .lastUpdated(Date.from(Instant.parse("2026-04-27T10:00:00.000Z"))) + .remoteAddr("remoteHost") + .build(); + + dsnBounce.service(mail); + + BodyPart dsnPart = (BodyPart) ((MimeMultipart) fakeMailContext.getSentMails() + .get(0).getMsg().getContent()).getBodyPart(1); + assertThat(dsnPart.getContentType()).startsWith("message/global-delivery-status"); + String body = IOUtils.toString((SharedByteArrayInputStream) dsnPart.getContent(), StandardCharsets.UTF_8); + assertThat(body).contains("Final-Recipient: utf-8; arnt@grå.org"); + } + } } \ No newline at end of file diff --git a/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/remote/delivery/SmtpUtf8StrategyTest.java b/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/remote/delivery/SmtpUtf8StrategyTest.java new file mode 100644 index 00000000000..a73f1d483ee --- /dev/null +++ b/server/mailet/mailets/src/test/java/org/apache/james/transport/mailets/remote/delivery/SmtpUtf8StrategyTest.java @@ -0,0 +1,194 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.transport.mailets.remote.delivery; + +import static org.assertj.core.api.Assertions.assertThat; + +import java.util.List; + +import jakarta.mail.internet.InternetAddress; + +import org.apache.james.core.MailAddress; +import org.apache.james.core.MaybeSender; +import org.junit.jupiter.api.Test; + +class SmtpUtf8StrategyTest { + + private static MaybeSender sender(String addr) throws Exception { + return MaybeSender.of(new MailAddress(addr)); + } + + private static List rcpts(String... addrs) throws Exception { + List out = new java.util.ArrayList<>(); + for (String a : addrs) { + InternetAddress ia = new InternetAddress(); + ia.setAddress(a); + out.add(ia); + } + return out; + } + + @Test + void envelopeNeedsUtf8ShouldBeFalseForAsciiEnvelope() throws Exception { + assertThat(SmtpUtf8Strategy.envelopeNeedsUtf8( + sender("arnt@example.com"), + rcpts("info@example.com"))) + .isFalse(); + } + + @Test + void envelopeNeedsUtf8ShouldBeTrueForUnicodeDomain() throws Exception { + assertThat(SmtpUtf8Strategy.envelopeNeedsUtf8( + sender("arnt@grå.org"), + rcpts("info@example.com"))) + .isTrue(); + } + + @Test + void envelopeNeedsUtf8ShouldBeTrueForUnicodeLocalPart() throws Exception { + assertThat(SmtpUtf8Strategy.envelopeNeedsUtf8( + sender("arnt@example.com"), + rcpts("réception@example.com"))) + .isTrue(); + } + + @Test + void envelopeNeedsUtf8ShouldBeFalseForNullSenderAndAsciiRecipient() throws Exception { + assertThat(SmtpUtf8Strategy.envelopeNeedsUtf8( + MaybeSender.nullSender(), + rcpts("info@example.com"))) + .isFalse(); + } + + @Test + void envelopeNeedsUtf8ShouldBeTrueForAceRecipientOnlyWhenNotYetEncoded() throws Exception { + // An address already in A-label form is pure ASCII: nothing to negotiate. + assertThat(SmtpUtf8Strategy.envelopeNeedsUtf8( + sender("arnt@example.com"), + rcpts("info@xn--gr-eka.org"))) + .isFalse(); + } + + @Test + void allAsciiShouldNotNeedUtf8() throws Exception { + assertThat(SmtpUtf8Strategy.pick( + sender("arnt@example.com"), + rcpts("info@example.com"), + /* remoteSupportsSmtpUtf8 */ false)) + .isEqualTo(SmtpUtf8Strategy.Action.NO_UTF8_NEEDED); + } + + @Test + void unicodeDomainWithSmtpUtf8ShouldUseExtension() throws Exception { + assertThat(SmtpUtf8Strategy.pick( + sender("arnt@grå.org"), + rcpts("info@grå.org"), + true)) + .isEqualTo(SmtpUtf8Strategy.Action.USE_EXTENSION); + } + + @Test + void unicodeLocalPartWithSmtpUtf8ShouldUseExtension() throws Exception { + assertThat(SmtpUtf8Strategy.pick( + sender("grå@example.com"), + rcpts("info@example.com"), + true)) + .isEqualTo(SmtpUtf8Strategy.Action.USE_EXTENSION); + } + + @Test + void unicodeDomainWithoutSmtpUtf8ShouldDowngradeDomains() throws Exception { + // ASCII local parts everywhere — we can ACE-encode the domain(s) + // and send RFC 5321-clean envelope commands. + assertThat(SmtpUtf8Strategy.pick( + sender("arnt@grå.org"), + rcpts("info@münchen.de"), + false)) + .isEqualTo(SmtpUtf8Strategy.Action.DOWNGRADE_DOMAINS); + } + + @Test + void unicodeLocalPartWithoutSmtpUtf8ShouldFailTransaction() throws Exception { + assertThat(SmtpUtf8Strategy.pick( + sender("grå@example.com"), + rcpts("info@example.com"), + false)) + .isEqualTo(SmtpUtf8Strategy.Action.CANNOT_DOWNGRADE); + } + + @Test + void nonAsciiInOnlyOneRecipientShouldStillTriggerAction() throws Exception { + assertThat(SmtpUtf8Strategy.pick( + sender("arnt@example.com"), + rcpts("info@example.com", "गोरिल@उदाहरण.भारत"), + false)) + .isEqualTo(SmtpUtf8Strategy.Action.CANNOT_DOWNGRADE); + } + + @Test + void nullSenderWithAsciiRecipientShouldNotNeedUtf8() throws Exception { + // Bounce path: MAIL FROM:<>. Only recipients matter. + assertThat(SmtpUtf8Strategy.pick( + MaybeSender.nullSender(), + rcpts("info@example.com"), + false)) + .isEqualTo(SmtpUtf8Strategy.Action.NO_UTF8_NEEDED); + } + + @Test + void nullSenderWithUnicodeRecipientShouldFollowRecipient() throws Exception { + assertThat(SmtpUtf8Strategy.pick( + MaybeSender.nullSender(), + rcpts("arnt@grå.org"), + true)) + .isEqualTo(SmtpUtf8Strategy.Action.USE_EXTENSION); + } + + @Test + void toAceDomainShouldConvertUnicodeDomain() throws Exception { + InternetAddress input = new InternetAddress(); + input.setAddress("arnt@grå.org"); + InternetAddress converted = SmtpUtf8Strategy.toAceDomain(input); + assertThat(converted.getAddress()).isEqualTo("arnt@xn--gr-zia.org"); + } + + @Test + void toAceDomainShouldLeaveAsciiDomainUntouched() throws Exception { + InternetAddress input = new InternetAddress(); + input.setAddress("arnt@example.com"); + InternetAddress converted = SmtpUtf8Strategy.toAceDomain(input); + assertThat(converted.getAddress()).isEqualTo("arnt@example.com"); + } + + @Test + void aceAddressStringShouldConvertDomainButPreserveLocalPart() { + // Local part "grå" is kept verbatim — this helper is only for the + // downgrade path, where callers have already confirmed the local + // part is ASCII. Preserving whatever local part arrived is the + // right contract. + assertThat(SmtpUtf8Strategy.aceAddressString("arnt@grå.org")) + .isEqualTo("arnt@xn--gr-zia.org"); + } + + @Test + void aceAddressStringShouldPreserveNullSender() { + assertThat(SmtpUtf8Strategy.aceAddressString("")).isEqualTo(""); + } +} diff --git a/server/mailet/remote-delivery-integration-testing/src/test/java/org/apache/james/smtp/utf8/SmtpUtf8RelayTest.java b/server/mailet/remote-delivery-integration-testing/src/test/java/org/apache/james/smtp/utf8/SmtpUtf8RelayTest.java new file mode 100644 index 00000000000..dd72349f44a --- /dev/null +++ b/server/mailet/remote-delivery-integration-testing/src/test/java/org/apache/james/smtp/utf8/SmtpUtf8RelayTest.java @@ -0,0 +1,184 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.smtp.utf8; + +import static org.apache.james.MemoryJamesServerMain.SMTP_AND_IMAP_MODULE; +import static org.apache.james.mailets.configuration.Constants.DEFAULT_DOMAIN; +import static org.apache.james.mailets.configuration.Constants.LOCALHOST_IP; +import static org.apache.james.mailets.configuration.Constants.calmlyAwait; +import static org.assertj.core.api.Assertions.assertThat; +import static org.awaitility.Durations.TEN_SECONDS; + +import java.io.File; + +import org.apache.commons.net.smtp.SMTPClient; +import org.apache.james.core.MailAddress; +import org.apache.james.dnsservice.api.DNSService; +import org.apache.james.dnsservice.api.InMemoryDNSService; +import org.apache.james.mailets.TemporaryJamesServer; +import org.apache.james.mailets.configuration.CommonProcessors; +import org.apache.james.mailets.configuration.MailetConfiguration; +import org.apache.james.mailets.configuration.MailetContainer; +import org.apache.james.mailets.configuration.ProcessorConfiguration; +import org.apache.james.mailets.configuration.SmtpConfiguration; +import org.apache.james.mock.smtp.server.model.Mail; +import org.apache.james.mock.smtp.server.model.SMTPExtension; +import org.apache.james.mock.smtp.server.model.SMTPExtensions; +import org.apache.james.mock.smtp.server.testing.MockSmtpServerExtension; +import org.apache.james.mock.smtp.server.testing.MockSmtpServerExtension.DockerMockSmtp; +import org.apache.james.modules.protocols.SmtpGuiceProbe; +import org.apache.james.transport.mailets.RecipientRewriteTable; +import org.apache.james.transport.mailets.RemoteDelivery; +import org.apache.james.transport.matchers.All; +import org.apache.james.utils.DataProbeImpl; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.extension.RegisterExtension; +import org.junit.jupiter.api.io.TempDir; + +/** + * End-to-end RFC 6531: a UTF-8 envelope enters over SMTPUTF8 and leaves through + * RemoteDelivery. Covers what {@code SmtpUtf8StrategyTest} cannot -- the strategy's + * verdict is one thing, what James actually puts on the wire is another. Angus + * reads {@code mail.mime.allowutf8} in the SMTPTransport constructor, so setting + * it after the transport is built silently disables the extension, and only a + * test at this level notices. + */ +class SmtpUtf8RelayTest { + private static final String ANOTHER_DOMAIN = "other.com"; + private static final String FROM = "from@" + DEFAULT_DOMAIN; + private static final String ASCII_RECIPIENT = "touser@" + ANOTHER_DOMAIN; + private static final String UTF8_RECIPIENT = "réception@" + ANOTHER_DOMAIN; + + @RegisterExtension + public static MockSmtpServerExtension mockSmtpExtension = new MockSmtpServerExtension(); + + private TemporaryJamesServer jamesServer; + + @BeforeEach + void setUp(@TempDir File temporaryFolder, DockerMockSmtp mockSmtp) throws Exception { + InMemoryDNSService inMemoryDNSService = new InMemoryDNSService() + .registerMxRecord(DEFAULT_DOMAIN, LOCALHOST_IP) + .registerMxRecord(ANOTHER_DOMAIN, mockSmtp.getIPAddress()); + + jamesServer = TemporaryJamesServer.builder() + .withBase(SMTP_AND_IMAP_MODULE) + .withOverrides(binder -> binder.bind(DNSService.class).toInstance(inMemoryDNSService)) + .withMailetContainer(MailetContainer.builder() + .putProcessor(CommonProcessors.simpleRoot()) + .putProcessor(CommonProcessors.error()) + .putProcessor(directResolutionTransport()) + .putProcessor(CommonProcessors.bounces())) + .withSmtpConfiguration(SmtpConfiguration.builder() + .withAutorizedAddresses("0.0.0.0/0.0.0.0")) + .build(temporaryFolder); + jamesServer.start(); + + jamesServer.getProbe(DataProbeImpl.class) + .fluent() + .addDomain(DEFAULT_DOMAIN); + } + + @AfterEach + void tearDown() { + jamesServer.shutdown(); + } + + private ProcessorConfiguration.Builder directResolutionTransport() { + return ProcessorConfiguration.transport() + .addMailet(MailetConfiguration.BCC_STRIPPER) + .addMailet(MailetConfiguration.builder() + .matcher(All.class) + .mailet(RecipientRewriteTable.class)) + .addMailet(MailetConfiguration.builder() + .mailet(RemoteDelivery.class) + .matcher(All.class) + .addProperty("outgoing", "outgoing") + .addProperty("delayTime", "3 * 10 ms") + .addProperty("maxRetries", "3") + .addProperty("deliveryThreads", "2") + .addProperty("sendpartial", "true")); + } + + @Test + void remoteDeliveryShouldAssertSmtpUtf8WhenRemoteAdvertisesIt(DockerMockSmtp mockSmtp) throws Exception { + mockSmtp.getConfigurationClient().setSMTPExtensions(SMTPExtensions.of(SMTPExtension.of("SMTPUTF8"))); + + sendWithSmtpUtf8(FROM, UTF8_RECIPIENT); + + calmlyAwait.atMost(TEN_SECONDS).untilAsserted(() -> assertThat(mockSmtp.getConfigurationClient().listMails()) + .hasSize(1) + .extracting(Mail::getEnvelope) + .containsExactly(Mail.Envelope.builder() + .from(new MailAddress(FROM)) + .addMailParameter(Mail.Parameter.builder() + .name("SMTPUTF8") + .build()) + .addRecipient(Mail.Recipient.builder() + .address(new MailAddress(UTF8_RECIPIENT)) + .build()) + .build())); + } + + @Test + void remoteDeliveryShouldNotAssertSmtpUtf8ForAnAsciiEnvelope(DockerMockSmtp mockSmtp) throws Exception { + mockSmtp.getConfigurationClient().setSMTPExtensions(SMTPExtensions.of(SMTPExtension.of("SMTPUTF8"))); + + sendWithSmtpUtf8(FROM, ASCII_RECIPIENT); + + calmlyAwait.atMost(TEN_SECONDS).untilAsserted(() -> assertThat(mockSmtp.getConfigurationClient().listMails()) + .hasSize(1) + .extracting(Mail::getEnvelope) + .containsExactly(Mail.Envelope.builder() + .from(new MailAddress(FROM)) + .addRecipient(Mail.Recipient.builder() + .address(new MailAddress(ASCII_RECIPIENT)) + .build()) + .build())); + } + + @Test + void remoteDeliveryShouldNotRelayAUnicodeLocalPartWhenRemoteLacksSmtpUtf8(DockerMockSmtp mockSmtp) throws Exception { + // No SMTPUTF8 advertised, and a non-ASCII local part has no lossless + // downgrade: the transaction must fail rather than mangle the address. + mockSmtp.getConfigurationClient().setSMTPExtensions(SMTPExtensions.of()); + + sendWithSmtpUtf8(FROM, UTF8_RECIPIENT); + + Thread.sleep(2000); + assertThat(mockSmtp.getConfigurationClient().listMails()).isEmpty(); + } + + private void sendWithSmtpUtf8(String from, String recipient) throws Exception { + SMTPClient smtpClient = new SMTPClient("UTF-8"); + try { + smtpClient.connect("localhost", jamesServer.getProbe(SmtpGuiceProbe.class).getSmtpPort().getValue()); + smtpClient.sendCommand("EHLO " + DEFAULT_DOMAIN); + smtpClient.sendCommand("MAIL FROM:<" + from + "> SMTPUTF8"); + assertThat(smtpClient.getReplyCode()).isEqualTo(250); + smtpClient.sendCommand("RCPT TO:<" + recipient + ">"); + assertThat(smtpClient.getReplyCode()).isEqualTo(250); + smtpClient.sendShortMessageData("From: " + from + "\r\nSubject: test\r\n\r\nbody\r\n.\r\n"); + } finally { + smtpClient.disconnect(); + } + } +} diff --git a/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapChannelUpstreamHandler.java b/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapChannelUpstreamHandler.java index 323745e4a07..2beeef9b6e6 100644 --- a/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapChannelUpstreamHandler.java +++ b/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapChannelUpstreamHandler.java @@ -431,7 +431,8 @@ public void channelRead(ChannelHandlerContext ctx, Object msg) { } ChannelImapResponseWriter writer = new ChannelImapResponseWriter(ctx.channel(), session); - ImapResponseComposerImpl response = new ImapResponseComposerImpl(writer); + ImapResponseComposerImpl response = new ImapResponseComposerImpl(writer) + .setUtf8Accepted(session.utf8Enabled()); writer.setFlushCallback(response::flush); ImapMessage message = (ImapMessage) msg; diff --git a/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java b/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java index 1a658c4a86e..676681367bc 100644 --- a/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java +++ b/server/protocols/protocols-imap4/src/main/java/org/apache/james/imapserver/netty/ImapRequestFrameDecoder.java @@ -146,6 +146,7 @@ private Optional parseImapMessage(ChannelHandlerContext ctx, ByteBu // Also check if the session was logged out if so there is not need to try to decode it. See JAMES-1341 if (session != null && session.getState() != ImapSessionState.LOGOUT) { try { + readerAndSize.getLeft().setUtf8Accept(session.utf8Enabled()); ImapMessage message = decoder.decode(readerAndSize.getLeft(), session); diff --git a/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerIDCommandTest.java b/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerIDCommandTest.java index a26b53c7efe..8fb6bbdbd31 100644 --- a/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerIDCommandTest.java +++ b/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerIDCommandTest.java @@ -24,6 +24,7 @@ import java.time.Duration; import org.apache.james.util.concurrency.ConcurrentTestRunner; +import org.apache.james.utils.TestIMAPClient; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.Test; @@ -61,15 +62,21 @@ void idCommandShouldReturnConfiguredResponse() throws Exception { } @Test - void concurrentIdCommandsInTheSameSessionShouldSucceed() throws Exception { + void concurrentIdCommandsShouldSucceed() throws Exception { imapServer = createImapServer("imapServer.xml"); + int port = imapServer.getListenAddresses().getFirst().getPort(); - testIMAPClient.connect("127.0.0.1", imapServer.getListenAddresses().getFirst().getPort()); + // One client per thread: a single TestIMAPClient wraps one socket and one + // reader, so sharing it across threads interleaves the replies and a thread + // can read an empty string that belongs to nobody. ConcurrentTestRunner.builder() .operation((threadNumber, step) -> { - assertThat(testIMAPClient.sendCommand("ID (\"name\" \"Apache James\")")) - .contains("* ID NIL") - .contains("OK ID completed."); + try (TestIMAPClient client = new TestIMAPClient()) { + client.connect("127.0.0.1", port); + assertThat(client.sendCommand("ID (\"name\" \"Apache James\")")) + .contains("* ID NIL") + .contains("OK ID completed."); + } }) .threadCount(20) .operationCount(1) diff --git a/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerSearchTest.java b/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerSearchTest.java index 884f35d4a33..7a5c629845a 100644 --- a/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerSearchTest.java +++ b/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerSearchTest.java @@ -21,26 +21,14 @@ import static org.apache.james.jmap.JMAPTestingConstants.LOCALHOST_IP; import static org.assertj.core.api.Assertions.assertThat; -import static org.assertj.core.api.Assertions.assertThatCode; import static org.assertj.core.api.Assertions.assertThatThrownBy; +import java.io.ByteArrayOutputStream; import java.io.IOException; import java.net.InetSocketAddress; import java.nio.ByteBuffer; import java.nio.channels.SocketChannel; import java.nio.charset.StandardCharsets; -import java.util.Properties; - -import jakarta.mail.Folder; -import jakarta.mail.Message; -import jakarta.mail.Session; -import jakarta.mail.Store; -import jakarta.mail.search.AndTerm; -import jakarta.mail.search.BodyTerm; -import jakarta.mail.search.FromStringTerm; -import jakarta.mail.search.RecipientStringTerm; -import jakarta.mail.search.SearchTerm; -import jakarta.mail.search.SubjectTerm; import org.apache.james.mailbox.MailboxSession; import org.apache.james.mailbox.MessageManager; @@ -303,34 +291,27 @@ void shouldRejectLongLiteralsWhenUnauthenticated() throws Exception { @Test void searchingShouldSupportMultipleUTF8Criteria() throws Exception { - String host = "127.0.0.1"; - Properties props = new Properties(); - props.put("mail.debug", "true"); - Session session = Session.getDefaultInstance(props, null); - Store store = session.getStore("imap"); - store.connect(host, port, USER.asString(), USER_PASS); - Folder folder = store.getFolder("INBOX"); - folder.open(Folder.READ_ONLY); - - SearchTerm subjectTerm = new SubjectTerm("java培训"); - SearchTerm fromTerm = new FromStringTerm("采购"); - SearchTerm recipientTerm = new RecipientStringTerm(Message.RecipientType.TO, "张三"); - SearchTerm ccRecipientTerm = new RecipientStringTerm(Message.RecipientType.CC, "李四"); - SearchTerm bccRecipientTerm = new RecipientStringTerm(Message.RecipientType.BCC, "王五"); - SearchTerm bodyTerm = new BodyTerm("天天向上"); - SearchTerm[] searchTerms = new SearchTerm[6]; - searchTerms[0] = subjectTerm; - searchTerms[1] = bodyTerm; - searchTerms[2] = fromTerm; - searchTerms[3] = recipientTerm; - searchTerms[4] = ccRecipientTerm; - searchTerms[5] = bccRecipientTerm; - SearchTerm andTerm = new AndTerm(searchTerms); - - assertThatCode(() -> folder.search(andTerm)).doesNotThrowAnyException(); - - folder.close(false); - store.close(); + MailboxSession mailboxSession = memoryIntegrationResources.getMailboxManager().createSystemSession(USER); + memoryIntegrationResources.getMailboxManager() + .createMailbox(MailboxPath.inbox(USER), mailboxSession); + + enableUtf8AndSelectInbox(); + + // Six UTF-8 criteria in one command, each as an RFC 6855 literal of UTF-8 + // octets. Driven over a raw socket rather than through jakarta.mail: once + // the server advertises UTF8=ACCEPT, angus-mail takes its supportsUtf8() + // branch and encodes search strings with ASCIIUtility.getBytes(), i.e. + // one truncated byte per char, so it never puts UTF-8 on the wire. + clientConnection.write(ByteBuffer.wrap(searchCommand("a3", + "SUBJECT", "java培训", + "BODY", "天天向上", + "FROM", "采购", + "TO", "张三", + "CC", "李四", + "BCC", "王五"))); + + assertThat(String.join("", readStringUntil(clientConnection, s -> s.contains("a3 ")))) + .contains("a3 OK"); } @Test @@ -353,20 +334,58 @@ void searchingASingleUTF8CriterionShouldComplete() throws Exception { "\r\n" + "

=E5=A4=A9=E5=A4=A9=E5=90=91=E4=B8=8A

\r\n"), mailboxSession); - String host = "127.0.0.1"; - Properties props = new Properties(); - props.put("mail.debug", "true"); - Session session = Session.getDefaultInstance(props, null); - Store store = session.getStore("imap"); - store.connect(host, port, USER.asString(), USER_PASS); - Folder folder = store.getFolder("INBOX"); - folder.open(Folder.READ_ONLY); + enableUtf8AndSelectInbox(); + + clientConnection.write(ByteBuffer.wrap(searchCommand("a3", "BODY", "天天向上"))); + + assertThat(String.join("", readStringUntil(clientConnection, s -> s.contains("a3 ")))) + .contains("* SEARCH 1") + .contains("a3 OK"); + } + + @Test + void searchingAUtf8QuotedStringShouldComplete() throws Exception { + MailboxSession mailboxSession = memoryIntegrationResources.getMailboxManager().createSystemSession(USER); + memoryIntegrationResources.getMailboxManager() + .createMailbox(MailboxPath.inbox(USER), mailboxSession); + memoryIntegrationResources.getMailboxManager() + .getMailbox(MailboxPath.inbox(USER), mailboxSession) + .appendMessage(MessageManager.AppendCommand.builder().build("Content-Type: text/plain; charset=UTF-8\r\n" + + "Subject: Test utf-8 charset\r\n" + + "\r\n" + + "天天向上\r\n"), mailboxSession); + + enableUtf8AndSelectInbox(); + + clientConnection.write(ByteBuffer.wrap("a3 SEARCH BODY \"天天向上\" ALL\r\n".getBytes(StandardCharsets.UTF_8))); - SearchTerm bodyTerm = new BodyTerm("天天向上"); + assertThat(String.join("", readStringUntil(clientConnection, s -> s.contains("a3 ")))) + .contains("* SEARCH 1") + .contains("a3 OK"); + } - assertThat(folder.search(bodyTerm)).hasSize(1); + private void enableUtf8AndSelectInbox() throws IOException { + clientConnection.write(ByteBuffer.wrap(String.format("a0 LOGIN %s %s\r\n", USER.asString(), USER_PASS).getBytes(StandardCharsets.UTF_8))); + readStringUntil(clientConnection, s -> s.contains("a0 OK")); + clientConnection.write(ByteBuffer.wrap("a1 ENABLE UTF8=ACCEPT\r\n".getBytes(StandardCharsets.UTF_8))); + readStringUntil(clientConnection, s -> s.contains("a1 OK")); + clientConnection.write(ByteBuffer.wrap("a2 SELECT INBOX\r\n".getBytes(StandardCharsets.UTF_8))); + readStringUntil(clientConnection, s -> s.contains("a2 OK")); + } - folder.close(false); - store.close(); + /** + * Builds {@code SEARCH {+}CRLF ... ALL CRLF}, + * the wire form a UTF8=ACCEPT client is meant to send for non-ASCII criteria. + */ + private byte[] searchCommand(String tag, String... keysAndValues) throws IOException { + ByteArrayOutputStream out = new ByteArrayOutputStream(); + out.write((tag + " SEARCH").getBytes(StandardCharsets.US_ASCII)); + for (int i = 0; i < keysAndValues.length; i += 2) { + byte[] value = keysAndValues[i + 1].getBytes(StandardCharsets.UTF_8); + out.write((" " + keysAndValues[i] + " {" + value.length + "+}\r\n").getBytes(StandardCharsets.US_ASCII)); + out.write(value); + } + out.write(" ALL\r\n".getBytes(StandardCharsets.US_ASCII)); + return out.toByteArray(); } } diff --git a/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerUtf8AcceptTest.java b/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerUtf8AcceptTest.java new file mode 100644 index 00000000000..c971ab14679 --- /dev/null +++ b/server/protocols/protocols-imap4/src/test/java/org/apache/james/imapserver/netty/IMAPServerUtf8AcceptTest.java @@ -0,0 +1,165 @@ +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.imapserver.netty; + +import static org.apache.james.jmap.JMAPTestingConstants.LOCALHOST_IP; +import static org.assertj.core.api.Assertions.assertThat; + +import java.io.IOException; +import java.net.InetSocketAddress; +import java.nio.ByteBuffer; +import java.nio.channels.SocketChannel; +import java.nio.charset.StandardCharsets; +import java.util.List; +import java.util.function.Predicate; + +import org.apache.james.mailbox.MailboxSession; +import org.apache.james.mailbox.model.MailboxPath; +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.Test; + +import com.google.common.collect.ImmutableList; + +class IMAPServerUtf8AcceptTest extends AbstractIMAPServerTest { + IMAPServer imapServer; + + @AfterEach + void tearDown() { + if (imapServer != null) { + imapServer.destroy(); + } + } + + @Test + void capabilityShouldAdvertiseUtf8Accept() throws Exception { + imapServer = createImapServer("imapServer.xml"); + assertThat( + testIMAPClient.connect("127.0.0.1", imapServer.getListenAddresses().getFirst().getPort()) + .sendCommand("CAPABILITY")) + .contains("UTF8=ACCEPT"); + } + + @Test + void enableUtf8AcceptShouldSucceed() throws Exception { + imapServer = createImapServer("imapServer.xml"); + assertThat( + testIMAPClient.connect("127.0.0.1", imapServer.getListenAddresses().getFirst().getPort()) + .login(USER.asString(), USER_PASS) + .sendCommand("ENABLE UTF8=ACCEPT")) + .contains("* ENABLED UTF8=ACCEPT") + .contains("OK ENABLE completed."); + } + + @Test + void enableUtf8AcceptShouldNotEchoUnsupportedCapability() throws Exception { + imapServer = createImapServer("imapServer.xml"); + assertThat( + testIMAPClient.connect("127.0.0.1", imapServer.getListenAddresses().getFirst().getPort()) + .login(USER.asString(), USER_PASS) + .sendCommand("ENABLE BOGUS-CAPABILITY UTF8=ACCEPT")) + .contains("* ENABLED UTF8=ACCEPT") + .doesNotContain("BOGUS-CAPABILITY") + .contains("OK ENABLE completed."); + } + + @Test + void listShouldEncodeMailboxNameAsModifiedUtf7WhenUtf8AcceptNotEnabled() throws Exception { + imapServer = createImapServer("imapServer.xml"); + MailboxSession session = memoryIntegrationResources.getMailboxManager().createSystemSession(USER); + memoryIntegrationResources.getMailboxManager() + .createMailbox(MailboxPath.forUser(USER, "grå"), session); + + try (SocketChannel c = SocketChannel.open(new InetSocketAddress(LOCALHOST_IP, + imapServer.getListenAddresses().getFirst().getPort()))) { + readUtf8Bytes(c); + c.write(ByteBuffer.wrap(String.format("a0 LOGIN %s %s\r\n", USER.asString(), USER_PASS).getBytes(StandardCharsets.UTF_8))); + readUtf8Until(c, s -> s.contains("a0 OK")); + c.write(ByteBuffer.wrap("a1 LIST \"\" \"*\"\r\n".getBytes(StandardCharsets.UTF_8))); + List replies = readUtf8Until(c, s -> s.contains("a1 OK")); + + assertThat(String.join("", replies)) + .contains("gr&AOU-") + .doesNotContain("grå"); + } + } + + @Test + void createWithUnicodeMailboxNameShouldSucceedAfterEnableUtf8Accept() throws Exception { + imapServer = createImapServer("imapServer.xml"); + + try (SocketChannel c = SocketChannel.open(new InetSocketAddress(LOCALHOST_IP, + imapServer.getListenAddresses().getFirst().getPort()))) { + readUtf8Bytes(c); + c.write(ByteBuffer.wrap(String.format("a0 LOGIN %s %s\r\n", USER.asString(), USER_PASS).getBytes(StandardCharsets.UTF_8))); + readUtf8Until(c, s -> s.contains("a0 OK")); + c.write(ByteBuffer.wrap("a1 ENABLE UTF8=ACCEPT\r\n".getBytes(StandardCharsets.UTF_8))); + readUtf8Until(c, s -> s.contains("a1 OK")); + c.write(ByteBuffer.wrap("a2 CREATE \"grå\"\r\n".getBytes(StandardCharsets.UTF_8))); + readUtf8Until(c, s -> s.contains("a2 OK")); + c.write(ByteBuffer.wrap("a3 LIST \"\" \"*\"\r\n".getBytes(StandardCharsets.UTF_8))); + List replies = readUtf8Until(c, s -> s.contains("a3 OK")); + + assertThat(String.join("", replies)).contains("grå"); + } + } + + @Test + void listShouldEncodeMailboxNameAsRawUtf8WhenUtf8AcceptEnabled() throws Exception { + imapServer = createImapServer("imapServer.xml"); + MailboxSession session = memoryIntegrationResources.getMailboxManager().createSystemSession(USER); + memoryIntegrationResources.getMailboxManager() + .createMailbox(MailboxPath.forUser(USER, "grå"), session); + + try (SocketChannel c = SocketChannel.open(new InetSocketAddress(LOCALHOST_IP, + imapServer.getListenAddresses().getFirst().getPort()))) { + readUtf8Bytes(c); + c.write(ByteBuffer.wrap(String.format("a0 LOGIN %s %s\r\n", USER.asString(), USER_PASS).getBytes(StandardCharsets.UTF_8))); + readUtf8Until(c, s -> s.contains("a0 OK")); + c.write(ByteBuffer.wrap("a1 ENABLE UTF8=ACCEPT\r\n".getBytes(StandardCharsets.UTF_8))); + readUtf8Until(c, s -> s.contains("a1 OK")); + c.write(ByteBuffer.wrap("a2 LIST \"\" \"*\"\r\n".getBytes(StandardCharsets.UTF_8))); + List replies = readUtf8Until(c, s -> s.contains("a2 OK")); + + assertThat(String.join("", replies)) + .contains("grå") + .doesNotContain("gr&AOU-"); + } + } + + private byte[] readUtf8Bytes(SocketChannel channel) throws IOException { + ByteBuffer buf = ByteBuffer.allocate(8192); + channel.read(buf); + buf.flip(); + byte[] out = new byte[buf.remaining()]; + buf.get(out); + return out; + } + + private List readUtf8Until(SocketChannel channel, Predicate condition) throws IOException { + ImmutableList.Builder result = ImmutableList.builder(); + while (true) { + String line = new String(readUtf8Bytes(channel), StandardCharsets.UTF_8); + result.add(line); + if (condition.test(line)) { + return result.build(); + } + } + } +} diff --git a/server/protocols/protocols-lmtp/src/main/java/org/apache/james/lmtpserver/CoreCmdHandlerLoader.java b/server/protocols/protocols-lmtp/src/main/java/org/apache/james/lmtpserver/CoreCmdHandlerLoader.java index aac876e098c..353956eff79 100644 --- a/server/protocols/protocols-lmtp/src/main/java/org/apache/james/lmtpserver/CoreCmdHandlerLoader.java +++ b/server/protocols/protocols-lmtp/src/main/java/org/apache/james/lmtpserver/CoreCmdHandlerLoader.java @@ -1,82 +1,84 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.lmtpserver; - -import java.util.List; - -import org.apache.james.lmtpserver.hook.MailboxDeliverToRecipientHandler; -import org.apache.james.protocols.api.handler.CommandDispatcher; -import org.apache.james.protocols.api.handler.CommandHandlerResultLogger; -import org.apache.james.protocols.lib.handler.HandlersPackage; -import org.apache.james.protocols.lmtp.core.LhloCmdHandler; -import org.apache.james.protocols.lmtp.core.WelcomeMessageHandler; -import org.apache.james.protocols.smtp.core.ExpnCmdHandler; -import org.apache.james.protocols.smtp.core.NoopCmdHandler; -import org.apache.james.protocols.smtp.core.PostmasterAbuseRcptHook; -import org.apache.james.protocols.smtp.core.QuitCmdHandler; -import org.apache.james.protocols.smtp.core.ReceivedDataLineFilter; -import org.apache.james.protocols.smtp.core.RsetCmdHandler; -import org.apache.james.protocols.smtp.core.VrfyCmdHandler; -import org.apache.james.protocols.smtp.core.esmtp.MailSizeEsmtpExtension; -import org.apache.james.protocols.smtp.core.esmtp.StartTlsCmdHandler; -import org.apache.james.protocols.smtp.core.log.HookResultLogger; -import org.apache.james.smtpserver.AuthRequiredToRelayRcptHook; -import org.apache.james.smtpserver.JamesDataCmdHandler; -import org.apache.james.smtpserver.JamesMailCmdHandler; -import org.apache.james.smtpserver.JamesRcptCmdHandler; -import org.apache.james.smtpserver.fastfail.ValidRcptHandler; - -/** - * This class represent the base command handlers which are shipped with james. - */ -public class CoreCmdHandlerLoader implements HandlersPackage { - - private static final List commands = List.of( - WelcomeMessageHandler.class.getName(), - CommandDispatcher.class.getName(), - JamesDataCmdHandler.class.getName(), - ExpnCmdHandler.class.getName(), - LhloCmdHandler.class.getName(), - JamesMailCmdHandler.class.getName(), - NoopCmdHandler.class.getName(), - QuitCmdHandler.class.getName(), - JamesRcptCmdHandler.class.getName(), - ValidRcptHandler.class.getName(), - RsetCmdHandler.class.getName(), - VrfyCmdHandler.class.getName(), - MailSizeEsmtpExtension.class.getName(), - StartTlsCmdHandler.class.getName(), - AuthRequiredToRelayRcptHook.class.getName(), - PostmasterAbuseRcptHook.class.getName(), - ReceivedDataLineFilter.class.getName(), - DataLineLMTPHandler.class.getName(), - MailboxDeliverToRecipientHandler.class.getName(), - CommandHandlerResultLogger.class.getName(), - HookResultLogger.class.getName() - ); - - public CoreCmdHandlerLoader() { - } - - @Override - public List getHandlers() { - return commands; - } -} +/**************************************************************** + * Licensed to the Apache Software Foundation (ASF) under one * + * or more contributor license agreements. See the NOTICE file * + * distributed with this work for additional information * + * regarding copyright ownership. The ASF licenses this file * + * to you under the Apache License, Version 2.0 (the * + * "License"); you may not use this file except in compliance * + * with the License. You may obtain a copy of the License at * + * * + * http://www.apache.org/licenses/LICENSE-2.0 * + * * + * Unless required by applicable law or agreed to in writing, * + * software distributed under the License is distributed on an * + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * + * KIND, either express or implied. See the License for the * + * specific language governing permissions and limitations * + * under the License. * + ****************************************************************/ + +package org.apache.james.lmtpserver; + +import java.util.List; + +import org.apache.james.lmtpserver.hook.MailboxDeliverToRecipientHandler; +import org.apache.james.protocols.api.handler.CommandDispatcher; +import org.apache.james.protocols.api.handler.CommandHandlerResultLogger; +import org.apache.james.protocols.lib.handler.HandlersPackage; +import org.apache.james.protocols.lmtp.core.LhloCmdHandler; +import org.apache.james.protocols.lmtp.core.WelcomeMessageHandler; +import org.apache.james.protocols.smtp.core.ExpnCmdHandler; +import org.apache.james.protocols.smtp.core.NoopCmdHandler; +import org.apache.james.protocols.smtp.core.PostmasterAbuseRcptHook; +import org.apache.james.protocols.smtp.core.QuitCmdHandler; +import org.apache.james.protocols.smtp.core.ReceivedDataLineFilter; +import org.apache.james.protocols.smtp.core.RsetCmdHandler; +import org.apache.james.protocols.smtp.core.VrfyCmdHandler; +import org.apache.james.protocols.smtp.core.esmtp.MailSizeEsmtpExtension; +import org.apache.james.protocols.smtp.core.esmtp.SMTPUTF8Extension; +import org.apache.james.protocols.smtp.core.esmtp.StartTlsCmdHandler; +import org.apache.james.protocols.smtp.core.log.HookResultLogger; +import org.apache.james.smtpserver.AuthRequiredToRelayRcptHook; +import org.apache.james.smtpserver.JamesDataCmdHandler; +import org.apache.james.smtpserver.JamesMailCmdHandler; +import org.apache.james.smtpserver.JamesRcptCmdHandler; +import org.apache.james.smtpserver.fastfail.ValidRcptHandler; + +/** + * This class represent the base command handlers which are shipped with james. + */ +public class CoreCmdHandlerLoader implements HandlersPackage { + + private static final List commands = List.of( + WelcomeMessageHandler.class.getName(), + CommandDispatcher.class.getName(), + JamesDataCmdHandler.class.getName(), + ExpnCmdHandler.class.getName(), + LhloCmdHandler.class.getName(), + JamesMailCmdHandler.class.getName(), + NoopCmdHandler.class.getName(), + QuitCmdHandler.class.getName(), + JamesRcptCmdHandler.class.getName(), + ValidRcptHandler.class.getName(), + RsetCmdHandler.class.getName(), + VrfyCmdHandler.class.getName(), + MailSizeEsmtpExtension.class.getName(), + SMTPUTF8Extension.class.getName(), + StartTlsCmdHandler.class.getName(), + AuthRequiredToRelayRcptHook.class.getName(), + PostmasterAbuseRcptHook.class.getName(), + ReceivedDataLineFilter.class.getName(), + DataLineLMTPHandler.class.getName(), + MailboxDeliverToRecipientHandler.class.getName(), + CommandHandlerResultLogger.class.getName(), + HookResultLogger.class.getName() + ); + + public CoreCmdHandlerLoader() { + } + + @Override + public List getHandlers() { + return commands; + } +} diff --git a/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/CoreCmdHandlerLoader.java b/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/CoreCmdHandlerLoader.java index 87d5e0e4927..12d0d69f035 100644 --- a/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/CoreCmdHandlerLoader.java +++ b/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/CoreCmdHandlerLoader.java @@ -37,6 +37,7 @@ import org.apache.james.protocols.smtp.core.esmtp.AuthCmdHandler; import org.apache.james.protocols.smtp.core.esmtp.EhloCmdHandler; import org.apache.james.protocols.smtp.core.esmtp.MailSizeEsmtpExtension; +import org.apache.james.protocols.smtp.core.esmtp.SMTPUTF8Extension; import org.apache.james.protocols.smtp.core.esmtp.StartTlsCmdHandler; import org.apache.james.protocols.smtp.core.log.HookResultLogger; @@ -61,6 +62,7 @@ public class CoreCmdHandlerLoader implements HandlersPackage { RsetCmdHandler.class.getName(), VrfyCmdHandler.class.getName(), MailSizeEsmtpExtension.class.getName(), + SMTPUTF8Extension.class.getName(), AuthRequiredToRelayRcptHook.class.getName(), SenderAuthIdentifyVerificationHook.class.getName(), AuthRequiredHook.class.getName(), diff --git a/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/smtputf8/SmtpUtf8RcptHook.java b/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/smtputf8/SmtpUtf8RcptHook.java deleted file mode 100644 index f8eeebbebf9..00000000000 --- a/server/protocols/protocols-smtp/src/main/java/org/apache/james/smtpserver/smtputf8/SmtpUtf8RcptHook.java +++ /dev/null @@ -1,50 +0,0 @@ -/**************************************************************** - * Licensed to the Apache Software Foundation (ASF) under one * - * or more contributor license agreements. See the NOTICE file * - * distributed with this work for additional information * - * regarding copyright ownership. The ASF licenses this file * - * to you under the Apache License, Version 2.0 (the * - * "License"); you may not use this file except in compliance * - * with the License. You may obtain a copy of the License at * - * * - * http://www.apache.org/licenses/LICENSE-2.0 * - * * - * Unless required by applicable law or agreed to in writing, * - * software distributed under the License is distributed on an * - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY * - * KIND, either express or implied. See the License for the * - * specific language governing permissions and limitations * - * under the License. * - ****************************************************************/ - -package org.apache.james.smtpserver.smtputf8; - -import java.util.Map; -import java.util.Set; - -import org.apache.james.core.MailAddress; -import org.apache.james.core.MaybeSender; -import org.apache.james.protocols.smtp.SMTPSession; -import org.apache.james.protocols.smtp.hook.HookResult; -import org.apache.james.protocols.smtp.hook.RcptHook; -import org.apache.mailet.Experimental; - -import com.google.common.collect.ImmutableSet; - -@Experimental -public class SmtpUtf8RcptHook implements RcptHook { - @Override - public Set supportedParameters() { - return ImmutableSet.of("SMTPUTF8"); - } - - @Override - public HookResult doRcpt(SMTPSession session, MaybeSender sender, MailAddress rcpt) { - return HookResult.DECLINED; - } - - @Override - public HookResult doRcpt(SMTPSession session, MaybeSender sender, MailAddress rcpt, Map parameters) { - return HookResult.DECLINED; - } -} diff --git a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/DSNTest.java b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/DSNTest.java index 1e0323da2ec..e3a5cf840ac 100644 --- a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/DSNTest.java +++ b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/DSNTest.java @@ -71,7 +71,7 @@ void ehloShouldAdvertiseDsnExtension() throws Exception { SoftAssertions.assertSoftly(softly -> { softly.assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); - softly.assertThat(smtpProtocol.getReplyString()).contains("250 DSN"); + softly.assertThat(smtpProtocol.getReplyString()).contains("250-DSN"); }); } diff --git a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/FutureReleaseTest.java b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/FutureReleaseTest.java index a8d2d727147..c719adace88 100644 --- a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/FutureReleaseTest.java +++ b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/FutureReleaseTest.java @@ -72,7 +72,7 @@ void ehloShouldAdvertiseFutureReleaseExtension() throws Exception { SoftAssertions.assertSoftly(softly -> { softly.assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); - softly.assertThat(smtpProtocol.getReplyString()).contains("250 FUTURERELEASE 86400 2023-04-15T10:00:00Z"); + softly.assertThat(smtpProtocol.getReplyString()).contains("250-FUTURERELEASE 86400 2023-04-15T10:00:00Z"); }); } diff --git a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SMTPServerTest.java b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SMTPServerTest.java index a62bea29d1c..29915c58f98 100644 --- a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SMTPServerTest.java +++ b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SMTPServerTest.java @@ -313,7 +313,10 @@ public void testSimpleMailSendWithEHLO() throws Exception { assertThat(capabilitieslist) .as("capabilities") - .hasSize(3); + .hasSize(4); + assertThat(capabilitieslist.contains("SMTPUTF8")) + .as("capabilities present SMTPUTF8") + .isTrue(); assertThat(capabilitieslist.contains("PIPELINING")) .as("capabilities present PIPELINING") .isTrue(); @@ -425,10 +428,10 @@ public void testStartTLSInEHLO() throws Exception { assertThat(capabilitieslist) .as("capabilities") - .hasSize(4); + .hasSize(5); assertThat(capabilitieslist) - .as("capabilities present PIPELINING ENHANCEDSTATUSCODES 8BITMIME STARTTLS") - .containsOnly("PIPELINING", "ENHANCEDSTATUSCODES", "8BITMIME", "STARTTLS"); + .as("capabilities present PIPELINING ENHANCEDSTATUSCODES 8BITMIME SMTPUTF8 STARTTLS") + .containsOnly("PIPELINING", "ENHANCEDSTATUSCODES", "8BITMIME", "SMTPUTF8", "STARTTLS"); smtpProtocol.quit(); smtpProtocol.disconnect(); diff --git a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpMtPriorityMessageHookTest.java b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpMtPriorityMessageHookTest.java index f75f27fbc8e..eb15cdbf091 100644 --- a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpMtPriorityMessageHookTest.java +++ b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpMtPriorityMessageHookTest.java @@ -64,7 +64,7 @@ void ehloShouldAdvertiseMtPriorityExtension() throws Exception { SoftAssertions.assertSoftly(softly -> { softly.assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); - softly.assertThat(smtpProtocol.getReplyString()).contains("250 MT-PRIORITY"); + softly.assertThat(smtpProtocol.getReplyString()).contains("250-MT-PRIORITY"); }); } diff --git a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpRequireTlsMessageHookTest.java b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpRequireTlsMessageHookTest.java index c259f04ebd7..cde8dcf366c 100644 --- a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpRequireTlsMessageHookTest.java +++ b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpRequireTlsMessageHookTest.java @@ -72,7 +72,7 @@ void ehloShouldAdvertiseRequireTlsExtension() throws Exception { SoftAssertions.assertSoftly(softly -> { softly.assertThat(client.getReplyCode()).isEqualTo(250); - softly.assertThat(client.getReplyString()).contains("250 REQUIRETLS"); + softly.assertThat(client.getReplyString()).contains("250-REQUIRETLS"); }); } diff --git a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpUtf8AnnounceTest.java b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpUtf8AnnounceTest.java index bf133afff36..d3cb39ffa5f 100644 --- a/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpUtf8AnnounceTest.java +++ b/server/protocols/protocols-smtp/src/test/java/org/apache/james/smtpserver/SmtpUtf8AnnounceTest.java @@ -19,27 +19,36 @@ package org.apache.james.smtpserver; +import static org.apache.james.smtpserver.SMTPServerTestSystem.LOCAL_DOMAIN; import static org.assertj.core.api.Assertions.assertThat; +import java.io.IOException; import java.net.InetSocketAddress; import org.apache.commons.net.smtp.SMTPClient; -import org.apache.james.server.core.configuration.FileConfigurationProvider; import org.apache.mailet.Mail; import org.assertj.core.api.SoftAssertions; import org.junit.jupiter.api.AfterEach; import org.junit.jupiter.api.BeforeEach; import org.junit.jupiter.api.Test; +/** + * RFC 6531 SMTPUTF8, exercised through a real {@link org.apache.james.smtpserver.netty.SMTPServer} + * and the stock handler chain, so that what is covered here is what an operator + * actually gets: {@code SMTPUTF8Extension} is pulled in by + * {@link CoreCmdHandlerLoader}, not wired by hand in the test configuration. + */ class SmtpUtf8AnnounceTest { + private static final String UTF8_SENDER = "expéditeur@remote.org"; + private static final String UTF8_RECIPIENT = "réception@" + LOCAL_DOMAIN; + /** RFC 6531 §4.2 rejection: 553 5.6.7. */ + private static final String NON_ASCII_WITHOUT_SMTPUTF8 = "553 5.6.7"; + private final SMTPServerTestSystem testSystem = new SMTPServerTestSystem(); @BeforeEach void setUp() throws Exception { - testSystem.preSetUp(); - testSystem.smtpServer.configure(FileConfigurationProvider.getConfig( - ClassLoader.getSystemResourceAsStream("smtpserver-utf8.xml"))); - testSystem.smtpServer.init(); + testSystem.setUp("smtpserver-noauth.xml"); } @AfterEach @@ -49,30 +58,87 @@ void tearDown() { @Test void ehloShouldAnnounceSmtpUtf8() throws Exception { - SMTPClient smtpProtocol = new SMTPClient(); - InetSocketAddress bindedAddress = testSystem.getBindedAddress(); - smtpProtocol.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + SMTPClient smtpProtocol = connect(); smtpProtocol.sendCommand("EHLO localhost"); SoftAssertions.assertSoftly(softly -> { softly.assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); - softly.assertThat(smtpProtocol.getReplyString()) - .contains("250-SMTPUTF8"); + softly.assertThat(smtpProtocol.getReplyString()).contains("SMTPUTF8"); }); } @Test - void trivialEmailWithSmtpUtf8ShouldBeReceived() throws Exception { - SMTPClient smtpProtocol = new SMTPClient(); - InetSocketAddress bindedAddress = testSystem.getBindedAddress(); - smtpProtocol.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + void ehloShouldAnnounceSmtpUtf8Once() throws Exception { + SMTPClient smtpProtocol = connect(); + smtpProtocol.sendCommand("EHLO localhost"); + + // EhloCmdHandler appends one line per EhloExtension without + // deduplicating, so a second handler advertising the keyword would + // silently produce a duplicate 250- line. + assertThat(smtpProtocol.getReplyString().split("SMTPUTF8", -1)).hasSize(2); + } + @Test + void unicodeAddressesShouldBeAcceptedWhenSmtpUtf8IsRequested() throws Exception { + SMTPClient smtpProtocol = connect(); smtpProtocol.sendCommand("EHLO remote.org"); - smtpProtocol.sendCommand("MAIL FROM: SMTPUTF8"); - smtpProtocol.sendCommand("RCPT TO: SMTPUTF8"); - smtpProtocol.sendShortMessageData("From: bob@localhost\r\n\r\nSubject: test mail\r\n\r\nTest body testSimpleMailSendWithDSN\r\n.\r\n"); + smtpProtocol.sendCommand("MAIL FROM: <" + UTF8_SENDER + "> SMTPUTF8"); + assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); + smtpProtocol.sendCommand("RCPT TO:<" + UTF8_RECIPIENT + ">"); + assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); + smtpProtocol.sendShortMessageData("From: " + UTF8_SENDER + "\r\nSubject: test\r\n\r\nbody\r\n.\r\n"); Mail lastMail = testSystem.queue.getLastMail(); assertThat(lastMail).isNotNull(); + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(lastMail.getMaybeSender().asString()).isEqualTo(UTF8_SENDER); + softly.assertThat(lastMail.getRecipients()) + .extracting(rcpt -> rcpt.asString()) + .containsExactly(UTF8_RECIPIENT); + }); + } + + @Test + void nonAsciiSenderShouldBeRejectedWithoutSmtpUtf8() throws Exception { + SMTPClient smtpProtocol = connect(); + smtpProtocol.sendCommand("EHLO remote.org"); + smtpProtocol.sendCommand("MAIL FROM: <" + UTF8_SENDER + ">"); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(smtpProtocol.getReplyCode()).isEqualTo(553); + softly.assertThat(smtpProtocol.getReplyString()).contains(NON_ASCII_WITHOUT_SMTPUTF8); + }); + } + + @Test + void nonAsciiRecipientShouldBeRejectedWithoutSmtpUtf8() throws Exception { + SMTPClient smtpProtocol = connect(); + smtpProtocol.sendCommand("EHLO remote.org"); + smtpProtocol.sendCommand("MAIL FROM: "); + assertThat(smtpProtocol.getReplyCode()).isEqualTo(250); + smtpProtocol.sendCommand("RCPT TO:<" + UTF8_RECIPIENT + ">"); + + SoftAssertions.assertSoftly(softly -> { + softly.assertThat(smtpProtocol.getReplyCode()).isEqualTo(553); + softly.assertThat(smtpProtocol.getReplyString()).contains(NON_ASCII_WITHOUT_SMTPUTF8); + }); + } + + @Test + void asciiEmailWithSmtpUtf8ShouldBeReceived() throws Exception { + SMTPClient smtpProtocol = connect(); + smtpProtocol.sendCommand("EHLO remote.org"); + smtpProtocol.sendCommand("MAIL FROM: SMTPUTF8"); + smtpProtocol.sendCommand("RCPT TO:"); + smtpProtocol.sendShortMessageData("From: bob@remote.org\r\nSubject: test\r\n\r\nbody\r\n.\r\n"); + + assertThat(testSystem.queue.getLastMail()).isNotNull(); + } + + private SMTPClient connect() throws IOException { + SMTPClient smtpProtocol = new SMTPClient("UTF-8"); + InetSocketAddress bindedAddress = testSystem.getBindedAddress(); + smtpProtocol.connect(bindedAddress.getAddress().getHostAddress(), bindedAddress.getPort()); + return smtpProtocol; } } diff --git a/server/protocols/protocols-smtp/src/test/resources/smtpserver-utf8.xml b/server/protocols/protocols-smtp/src/test/resources/smtpserver-utf8.xml deleted file mode 100644 index 6be818709d1..00000000000 --- a/server/protocols/protocols-smtp/src/test/resources/smtpserver-utf8.xml +++ /dev/null @@ -1,52 +0,0 @@ - - - - - - - - 0.0.0.0:0 - 200 - - file://conf/keystore - james72laBalle - org.bouncycastle.jce.provider.BouncyCastleProvider - SunX509 - - 360 - 0 - 0 - - forUnauthorizedAddresses - false - - true - 0 - true - Apache JAMES awesome SMTP Server - - - - - - false - - - diff --git a/server/testing/src/main/java/org/apache/james/utils/SMTPMessageSender.java b/server/testing/src/main/java/org/apache/james/utils/SMTPMessageSender.java index 9ee8ef6f355..21dfd2e2b75 100644 --- a/server/testing/src/main/java/org/apache/james/utils/SMTPMessageSender.java +++ b/server/testing/src/main/java/org/apache/james/utils/SMTPMessageSender.java @@ -138,6 +138,22 @@ public SMTPMessageSender sendMessageWithHeaders(String from, List recipi return this; } + /** + * Opens the transaction with EHLO and asserts the SMTPUTF8 extension (RFC 6531), which is + * required whenever the envelope carries non-ASCII addresses. + */ + public SMTPMessageSender sendMessageWithHeadersSmtpUtf8(String from, String recipient, String message) throws IOException { + return sendMessageWithHeadersSmtpUtf8(from, ImmutableList.of(recipient), message); + } + + public SMTPMessageSender sendMessageWithHeadersSmtpUtf8(String from, List recipients, String message) throws IOException { + doEhlo(); + doSetSenderSmtpUtf8(from); + recipients.forEach(Throwing.consumer(this::doAddRcpt).sneakyThrow()); + doData(message); + return this; + } + public SMTPMessageSender sendMessageNoSender(String from, String recipient) throws IOException { doHelo(); doSetSender(""); @@ -186,6 +202,20 @@ private void doSetSender(String from) throws IOException { } } + private void doSetSenderSmtpUtf8(String from) throws IOException { + int code = smtpClient.mail("<" + from + "> SMTPUTF8"); + if (code != 250) { + throw new SMTPSendingException(SmtpSendingStep.Sender, smtpClient.getReplyString()); + } + } + + private void doEhlo() throws IOException { + int code = smtpClient.ehlo(senderDomain); + if (code != 250) { + throw new SMTPSendingException(SmtpSendingStep.Helo, smtpClient.getReplyString()); + } + } + private void doHelo() throws IOException { int code = smtpClient.helo(senderDomain); if (code != 250) { diff --git a/server/testing/src/main/java/org/apache/james/utils/TestIMAPClient.java b/server/testing/src/main/java/org/apache/james/utils/TestIMAPClient.java index 048da9a73ad..43be854c9ed 100644 --- a/server/testing/src/main/java/org/apache/james/utils/TestIMAPClient.java +++ b/server/testing/src/main/java/org/apache/james/utils/TestIMAPClient.java @@ -50,30 +50,68 @@ public class TestIMAPClient extends ExternalResource implements Closeable, After private static final int MESSAGE_NUMBER_MATCHING_GROUP = 1; public static final String INBOX = "INBOX"; - public static class Utf8IMAPSClient extends AuthenticatingIMAPClient { + /** + * commons-net announces and consumes IMAP literals in octets, but subtracts the + * {@link String#length()} of the lines it has decoded to know when a literal is over. Its + * streams therefore have to stay octet transparent - one char per octet, which is what its + * own ISO-8859-1 default gives. Decoding the socket as UTF-8 makes every multi-byte + * character count for one octet less than the server announced, so the client keeps reading + * past the literal, swallows the tagged reply as if it were message content and then blocks + * forever waiting for a completion line that has already gone by. + * + * UTF-8 is handled at {@link TestIMAPClient}'s own boundary instead: see + * {@link TestIMAPClient#asOctets(String)} and {@link TestIMAPClient#asText(String)}. + */ + public static class OctetIMAPClient extends AuthenticatingIMAPClient { @Override protected void _connectAction_() throws IOException { super._connectAction_(); - _reader = new CRLFLineReader(new InputStreamReader(_input_, StandardCharsets.UTF_8)); - __writer = new BufferedWriter(new OutputStreamWriter(_output_, StandardCharsets.UTF_8)); + _reader = new CRLFLineReader(new InputStreamReader(_input_, StandardCharsets.ISO_8859_1)); + __writer = new BufferedWriter(new OutputStreamWriter(_output_, StandardCharsets.ISO_8859_1)); } } + /** + * Turns text into the octets to put on the wire: one char per UTF-8 octet, as + * {@link OctetIMAPClient} expects. + */ + private static String asOctets(String text) { + return new String(text.getBytes(StandardCharsets.UTF_8), StandardCharsets.ISO_8859_1); + } + + /** + * Reverse of {@link #asOctets(String)}: reads back the octets commons-net collected as UTF-8 + * text. + */ + private static String asText(String octets) { + return new String(octets.getBytes(StandardCharsets.ISO_8859_1), StandardCharsets.UTF_8); + } + private final IMAPClient imapClient; @VisibleForTesting - TestIMAPClient(Utf8IMAPSClient imapClient) { + TestIMAPClient(OctetIMAPClient imapClient) { this.imapClient = imapClient; } public TestIMAPClient() { - this(new Utf8IMAPSClient()); + this(new OctetIMAPClient()); } public TestIMAPClient(IMAPClient imapClient) { this.imapClient = imapClient; } + private String replyString() { + return asText(imapClient.getReplyString()); + } + + private List replyStrings() { + return Stream.of(imapClient.getReplyStrings()) + .map(TestIMAPClient::asText) + .collect(ImmutableList.toImmutableList()); + } + public TestIMAPClient connect(String host, int port) throws IOException { imapClient.connect(host, port); return this; @@ -81,7 +119,7 @@ public TestIMAPClient connect(String host, int port) throws IOException { public String capability() throws IOException { imapClient.capability(); - return imapClient.getReplyString(); + return replyString(); } public TestIMAPClient disconnect() throws IOException { @@ -90,7 +128,7 @@ public TestIMAPClient disconnect() throws IOException { } public TestIMAPClient login(String user, String password) throws IOException { - final boolean login = imapClient.login(user, password); + final boolean login = imapClient.login(asOctets(user), asOctets(password)); if (!login) { throw new IOException("Login failed"); } @@ -107,9 +145,9 @@ public TestIMAPClient authenticatePlain(String user, String password) throws Exc } public TestIMAPClient rawLogin(String user, String password) throws IOException { - imapClient.sendCommand("LOGIN " + user + " " + password); + imapClient.sendCommand(asOctets("LOGIN " + user + " " + password)); - if (imapClient.getReplyString().contains("NO LOGIN failed.")) { + if (replyString().contains("NO LOGIN failed.")) { throw new IOException("Login failed"); } return this; @@ -117,7 +155,7 @@ public TestIMAPClient rawLogin(String user, String password) throws IOException public List list() throws IOException { imapClient.list("", "*"); - return ImmutableList.copyOf(imapClient.getReplyStrings()); + return replyStrings(); } public TestIMAPClient login(Username user, String password) throws IOException { @@ -125,13 +163,13 @@ public TestIMAPClient login(Username user, String password) throws IOException { } public TestIMAPClient select(String mailbox) throws IOException { - imapClient.select(mailbox); + imapClient.select(asOctets(mailbox)); return this; } public TestIMAPClient create(String mailbox) throws IOException { - if (!imapClient.create(mailbox)) { - throw new RuntimeException(imapClient.getReplyString()); + if (!imapClient.create(asOctets(mailbox))) { + throw new RuntimeException(replyString()); } return this; } @@ -139,20 +177,20 @@ public TestIMAPClient create(String mailbox) throws IOException { public TestIMAPClient append(String mailboxName, String message) throws IOException { String noFlags = null; String noDateTime = null; - if (!imapClient.append(mailboxName, noFlags, noDateTime, message)) { - throw new RuntimeException(imapClient.getReplyString()); + if (!imapClient.append(asOctets(mailboxName), noFlags, noDateTime, asOctets(message))) { + throw new RuntimeException(replyString()); } return this; } public TestIMAPClient delete(String mailbox) throws IOException { - imapClient.delete(mailbox); + imapClient.delete(asOctets(mailbox)); return this; } public boolean hasAMessage() throws IOException { imapClient.fetch("1", "UID"); - return imapClient.getReplyString() + return replyString() .contains("OK FETCH completed"); } @@ -172,7 +210,7 @@ public TestIMAPClient awaitMessageCount(ConditionFactory conditionFactory, int m private long countFetchedEntries() { return Splitter.on("\n") .trimResults() - .splitToStream(imapClient.getReplyString()) + .splitToStream(replyString()) .filter(s -> s.startsWith("*")) .count(); } @@ -184,8 +222,7 @@ public TestIMAPClient awaitNoMessage(ConditionFactory conditionFactory) { public boolean hasAMessageWithFlags(String flags) throws IOException { imapClient.fetch("1:1", "ALL"); - String replyString = imapClient.getReplyString(); - return isCompletedWithFlags(flags, replyString); + return isCompletedWithFlags(flags, replyString()); } @VisibleForTesting @@ -197,12 +234,12 @@ boolean isCompletedWithFlags(String flags, String replyString) { } public boolean userGetNotifiedForNewMessagesWhenSelectingMailbox(int numOfNewMessage) { - return imapClient.getReplyString().contains("OK [UNSEEN " + numOfNewMessage + "]"); + return replyString().contains("OK [UNSEEN " + numOfNewMessage + "]"); } public boolean userDoesNotReceiveMessage() throws IOException { imapClient.fetch("1:1", "ALL"); - return imapClient.getReplyString() + return replyString() .contains("BAD FETCH failed. Invalid messageset"); } @@ -216,27 +253,26 @@ public String readFirstMessageHeaders() throws IOException { public String setFlagsForAllMessagesInMailbox(String flag) throws IOException { imapClient.store("1:*", "+FLAGS", flag); - return imapClient.getReplyString(); + return replyString(); } public String copyAllMessagesInMailboxTo(String mailboxName) throws IOException { - imapClient.copy("1:*", mailboxName); - return imapClient.getReplyString(); + imapClient.copy("1:*", asOctets(mailboxName)); + return replyString(); } public String readFirstMessageInMailbox(String parameters) throws IOException { imapClient.fetch("1:1", parameters); - return imapClient.getReplyString(); + return replyString(); } public boolean userGetNotifiedForNewMessages(int numberOfMessages) throws IOException { imapClient.noop(); - String replyString = imapClient.getReplyString(); List parts = Splitter.on('\n') .trimResults() .omitEmptyStrings() - .splitToList(replyString); + .splitToList(replyString()); return parts.size() == 3 && parts.get(2).contains("OK NOOP completed.") && parts.contains("* " + numberOfMessages + " EXISTS") @@ -246,11 +282,10 @@ public boolean userGetNotifiedForNewMessages(int numberOfMessages) throws IOExce public boolean userGetNotifiedForDeletion(int msn) throws IOException { imapClient.noop(); - String replyString = imapClient.getReplyString(); List parts = Splitter.on('\n') .trimResults() .omitEmptyStrings() - .splitToList(replyString); + .splitToList(replyString()); return parts.size() == 2 && parts.get(1).contains("OK NOOP completed.") @@ -279,11 +314,11 @@ public void afterEach(ExtensionContext extensionContext) { } public void copyFirstMessage(String destMailbox) throws IOException { - imapClient.copy("1", destMailbox); + imapClient.copy("1", asOctets(destMailbox)); } public void moveFirstMessage(String destMailbox) throws IOException { - imapClient.sendCommand("MOVE 1 " + destMailbox); + imapClient.sendCommand(asOctets("MOVE 1 " + destMailbox)); } public void expunge() throws IOException { @@ -291,18 +326,18 @@ public void expunge() throws IOException { } public String getQuotaRoot(String mailbox) throws IOException { - imapClient.sendCommand("GETQUOTAROOT " + mailbox); - return imapClient.getReplyString(); + imapClient.sendCommand(asOctets("GETQUOTAROOT " + mailbox)); + return replyString(); } public String sendCommand(String command) throws IOException { - imapClient.sendCommand(command); - return imapClient.getReplyString(); + imapClient.sendCommand(asOctets(command)); + return replyString(); } public long getMessageCount(String mailboxName) throws IOException { - imapClient.examine(mailboxName); - return Stream.of(imapClient.getReplyStrings()) + imapClient.examine(asOctets(mailboxName)); + return replyStrings().stream() .map(EXAMINE_EXISTS::matcher) .filter(Matcher::matches) .map(m -> m.group(MESSAGE_NUMBER_MATCHING_GROUP)) diff --git a/server/testing/src/test/java/org/apache/james/utils/TestIMAPClientTest.java b/server/testing/src/test/java/org/apache/james/utils/TestIMAPClientTest.java index 8bf5802d876..b4a8cdc1b4f 100644 --- a/server/testing/src/test/java/org/apache/james/utils/TestIMAPClientTest.java +++ b/server/testing/src/test/java/org/apache/james/utils/TestIMAPClientTest.java @@ -21,11 +21,11 @@ import static org.assertj.core.api.Assertions.assertThat; -import org.apache.james.utils.TestIMAPClient.Utf8IMAPSClient; +import org.apache.james.utils.TestIMAPClient.OctetIMAPClient; import org.junit.jupiter.api.Test; class TestIMAPClientTest { - static final Utf8IMAPSClient NULL_IMAP_CLIENT = null; + static final OctetIMAPClient NULL_IMAP_CLIENT = null; TestIMAPClient testee = new TestIMAPClient(NULL_IMAP_CLIENT); @Test