From c988f37a42b8e6c7d50379cb207831bf49492ca9 Mon Sep 17 00:00:00 2001 From: Shai Almog <67850168+shai-almog@users.noreply.github.com> Date: Wed, 11 Nov 2026 18:33:02 +0200 Subject: [PATCH] Backend: email -- SMTP and SES, templates, an outbox, and the security flows on top A mail API for the backend runtime (com.codename1.backend.mail): MailSender and an immutable Mail, MIME written once (quoted-printable, wrapped Base64, encoded words, multipart nesting), and transports chosen by cn1.mail.url -- smtp:// and smtps:// (STARTTLS, PLAIN/LOGIN/XOAUTH2, pipelining, a small connection pool), ses:// over the signed API, memory: for tests and log: for development. Only the transport the application names is linked. With a data source a mail is staged in cn1_mail_outbox inside the caller's transaction and sent after commit, retried with back-off, and buried in cn1_mail_dead when it is refused for good. Templates under templates/mail render through the generated views. Security: password reset, email verification and a mailed sign-in code as a second factor, all on single-use hashed tokens. The generated login page links to the reset page when one is configured. MailCheck runs one body on the JVM and natively against an in-process SMTP server, and on SQLite, PostgreSQL and MySQL; BackendMailTest runs it in CI. Co-Authored-By: Claude Fable 5.2 --- .../developerguide/backend/mail/Receipts.java | 82 ++ .../developerguide/backend/mail/Signup.java | 55 + .../backend/mail/SignupTest.java | 60 + .../backend/security/MailSecurityConfig.java | 108 ++ .../src/main/resources/application.properties | 5 + .../src/main/snippets/mail-shipped.html | 9 + docs/developer-guide/Backend-Mail.asciidoc | 294 ++++ .../developer-guide/Backend-Security.asciidoc | 129 ++ docs/developer-guide/Backend-Testing.asciidoc | 9 + docs/developer-guide/developer-guide.asciidoc | 2 + .../backend/security/MailFlowsTest.java | 492 +++++++ .../security/SecuritySchemaHistoryTest.java | 14 +- .../backend/security/SecurityStoresTest.java | 7 +- .../maven/processors/BackendBeans.java | 86 ++ .../maven/processors/BackendWiringWriter.java | 36 + .../maven/processors/MvcTemplates.java | 20 + .../RestControllerAnnotationProcessor.java | 12 +- .../maven/processors/BackendBeansTest.java | 46 + .../processors/MigrationGeneratorTest.java | 12 +- .../demo/mailcheck/com/demo/MailCheck.java | 1288 +++++++++++++++++ .../demo/mailcheck/com/demo/MailFlows.java | 366 +++++ .../demo/mailcheck/com/demo/SesCheck.java | 289 ++++ .../com/codename1/backend/mail/Address.java | 142 ++ .../com/codename1/backend/mail/HtmlText.java | 336 +++++ .../codename1/backend/mail/LogTransport.java | 63 + .../src/com/codename1/backend/mail/Mail.java | 551 +++++++ .../mail/MailAuthenticationException.java | 36 + .../codename1/backend/mail/MailException.java | 46 + .../codename1/backend/mail/MailMetrics.java | 109 ++ .../com/codename1/backend/mail/MailQueue.java | 614 ++++++++ .../backend/mail/MailSendException.java | 50 + .../codename1/backend/mail/MailSender.java | 93 ++ .../codename1/backend/mail/MailTransport.java | 58 + .../com/codename1/backend/mail/Mailer.java | 207 +++ .../com/codename1/backend/mail/Mailers.java | 250 ++++ .../backend/mail/MemoryTransport.java | 43 + .../codename1/backend/mail/MimeWriter.java | 559 +++++++ .../codename1/backend/mail/package-info.java | 34 + .../backend/mail/ses/SesTransport.java | 246 ++++ .../backend/mail/ses/package-info.java | 26 + .../backend/mail/smtp/SmtpConnection.java | 560 +++++++ .../backend/mail/smtp/SmtpTransport.java | 295 ++++ .../backend/mail/smtp/package-info.java | 26 + .../backend/messaging/OutboxTransport.java | 12 + .../DefaultLoginPageGeneratingFilter.java | 16 +- .../security/EmailVerificationConfigurer.java | 109 ++ .../security/EmailVerificationFilter.java | 90 ++ .../backend/security/FormLoginConfigurer.java | 2 +- .../backend/security/HttpSecurity.java | 47 + .../backend/security/MfaConfigurer.java | 25 +- .../security/PasswordResetConfigurer.java | 187 +++ .../backend/security/PasswordResetFilter.java | 227 +++ .../SecondFactorAuthenticationFilter.java | 78 +- .../backend/security/SecuritySchema.java | 28 +- .../security/mail/EmailAddressResolver.java | 49 + .../mail/EmailVerificationService.java | 220 +++ .../mail/InMemoryOneTimeTokenRepository.java | 83 ++ .../mail/InMemoryVerifiedEmailRepository.java | 79 + .../mail/JdbcOneTimeTokenRepository.java | 136 ++ .../mail/JdbcVerifiedEmailRepository.java | 158 ++ .../backend/security/mail/MailText.java | 74 + .../security/mail/OneTimeTokenRepository.java | 48 + .../security/mail/OneTimeTokenService.java | 101 ++ .../security/mail/PasswordResetService.java | 231 +++ .../mail/VerifiedEmailRepository.java | 56 + .../backend/security/mail/package-info.java | 41 + .../backend/security/mfa/EmailCode.java | 66 + .../security/mfa/EmailCodeRepository.java | 60 + .../security/mfa/EmailCodeService.java | 279 ++++ .../mfa/InMemoryEmailCodeRepository.java | 83 ++ .../security/mfa/JdbcEmailCodeRepository.java | 170 +++ .../codename1/impl/backend/MailRuntime.java | 91 ++ .../codename1/impl/backend/Management.java | 17 + .../tools/translator/BackendMailTest.java | 132 ++ 74 files changed, 10723 insertions(+), 37 deletions(-) create mode 100644 docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java create mode 100644 docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Signup.java create mode 100644 docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/SignupTest.java create mode 100644 docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java create mode 100644 docs/demos/backend/src/main/snippets/mail-shipped.html create mode 100644 docs/developer-guide/Backend-Mail.asciidoc create mode 100644 maven/backend/src/test/java/com/codename1/backend/security/MailFlowsTest.java create mode 100644 vm/backend/demo/mailcheck/com/demo/MailCheck.java create mode 100644 vm/backend/demo/mailcheck/com/demo/MailFlows.java create mode 100644 vm/backend/demo/mailcheck/com/demo/SesCheck.java create mode 100644 vm/backend/src/com/codename1/backend/mail/Address.java create mode 100644 vm/backend/src/com/codename1/backend/mail/HtmlText.java create mode 100644 vm/backend/src/com/codename1/backend/mail/LogTransport.java create mode 100644 vm/backend/src/com/codename1/backend/mail/Mail.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailAuthenticationException.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailException.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailMetrics.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailQueue.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailSendException.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailSender.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MailTransport.java create mode 100644 vm/backend/src/com/codename1/backend/mail/Mailer.java create mode 100644 vm/backend/src/com/codename1/backend/mail/Mailers.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MemoryTransport.java create mode 100644 vm/backend/src/com/codename1/backend/mail/MimeWriter.java create mode 100644 vm/backend/src/com/codename1/backend/mail/package-info.java create mode 100644 vm/backend/src/com/codename1/backend/mail/ses/SesTransport.java create mode 100644 vm/backend/src/com/codename1/backend/mail/ses/package-info.java create mode 100644 vm/backend/src/com/codename1/backend/mail/smtp/SmtpConnection.java create mode 100644 vm/backend/src/com/codename1/backend/mail/smtp/SmtpTransport.java create mode 100644 vm/backend/src/com/codename1/backend/mail/smtp/package-info.java create mode 100644 vm/backend/src/com/codename1/backend/security/EmailVerificationConfigurer.java create mode 100644 vm/backend/src/com/codename1/backend/security/EmailVerificationFilter.java create mode 100644 vm/backend/src/com/codename1/backend/security/PasswordResetConfigurer.java create mode 100644 vm/backend/src/com/codename1/backend/security/PasswordResetFilter.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/EmailAddressResolver.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/EmailVerificationService.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/InMemoryOneTimeTokenRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/InMemoryVerifiedEmailRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/JdbcOneTimeTokenRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/JdbcVerifiedEmailRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/MailText.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenService.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/PasswordResetService.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/VerifiedEmailRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mail/package-info.java create mode 100644 vm/backend/src/com/codename1/backend/security/mfa/EmailCode.java create mode 100644 vm/backend/src/com/codename1/backend/security/mfa/EmailCodeRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mfa/EmailCodeService.java create mode 100644 vm/backend/src/com/codename1/backend/security/mfa/InMemoryEmailCodeRepository.java create mode 100644 vm/backend/src/com/codename1/backend/security/mfa/JdbcEmailCodeRepository.java create mode 100644 vm/backend/src/com/codename1/impl/backend/MailRuntime.java create mode 100644 vm/tests/src/test/java/com/codename1/tools/translator/BackendMailTest.java diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java new file mode 100644 index 00000000000..1bfc3337f53 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java @@ -0,0 +1,82 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend.mail; + +import com.codename1.backend.annotations.Component; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailException; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.mvc.Model; +import com.codenameone.developerguide.backend.orders.Order; + +import java.io.IOException; + +@Component +public class Receipts { + private final MailSender mail; + + public Receipts(MailSender mail) { + this.mail = mail; + } + + // tag::backend-mail-template[] + public void shipped(Order order, String email) throws IOException { + Model model = new Model() + .addAttribute("customer", order.customer) + .addAttribute("orderId", Long.valueOf(order.id)); + mail.send(Mail.to(email) + .subject("Your order shipped") + .template("mail/shipped", model) + .build()); + } + // end::backend-mail-template[] + + // tag::backend-mail-attach[] + public void invoice(Order order, String email, byte[] pdf, byte[] logo) throws IOException { + mail.send(Mail.to(email) + .cc("accounts@example.com") + .replyTo("billing@example.com") + .subject("Invoice " + order.id) + .html("\"Acme\"

Your invoice is attached.

") + .inline("logo", "image/png", logo) + .attach("invoice-" + order.id + ".pdf", "application/pdf", pdf) + .header("X-Order", String.valueOf(order.id)) + .build()); + } + // end::backend-mail-attach[] + + // tag::backend-mail-now[] + public boolean sendCode(String email, String code) throws IOException { + try { + mail.sendNow(Mail.to(email) + .subject("Your code") + .text("Your code is " + code) + .build()); + return true; + } catch (MailException refused) { + // isPermanent(): the server will never take this mail. + return false; + } + } + // end::backend-mail-now[] +} diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Signup.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Signup.java new file mode 100644 index 00000000000..4eb308b4240 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Signup.java @@ -0,0 +1,55 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend.mail; + +import com.codename1.backend.DataSource; +import com.codename1.backend.annotations.Component; +import com.codename1.backend.annotations.Transactional; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; + +import java.io.IOException; + +// tag::backend-mail-send[] +@Component +public class Signup { + private final DataSource db; + private final MailSender mail; + + public Signup(DataSource db, MailSender mail) { + this.db = db; + this.mail = mail; + } + + @Transactional + public void register(String name, String email) throws IOException { + db.execute("INSERT INTO accounts (name, email) VALUES (?, ?)", + new Object[] {name, email}); + // Staged in this transaction, sent once it commits. + mail.send(Mail.to(email) + .subject("Welcome, " + name) + .text("Your account is ready. Sign in at https://example.com/login") + .build()); + } +} +// end::backend-mail-send[] diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/SignupTest.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/SignupTest.java new file mode 100644 index 00000000000..e28cd679ed3 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/SignupTest.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend.mail; + +import com.codename1.backend.annotations.Autowired; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.test.BackendTest; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; + +// tag::backend-mail-test[] +@BackendTest(properties = "cn1.mail.from=Acme ") +class SignupTest { + @Autowired + private Signup signup; + + @Autowired + private MailSender mail; + + @BeforeEach + void emptyInbox() throws Exception { + mail.clearSent(); + } + + @Test + void aNewAccountIsWelcomed() throws Exception { + signup.register("Ada", "ada@example.com"); + + assertEquals(1, mail.sent().size()); + Mail welcome = mail.sent().get(0); + assertEquals("ada@example.com", welcome.getTo().get(0)); + assertEquals("Welcome, Ada", welcome.getSubject()); + assertTrue(welcome.getText().contains("https://example.com/login")); + } +} +// end::backend-mail-test[] diff --git a/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java new file mode 100644 index 00000000000..f205e2f72a6 --- /dev/null +++ b/docs/demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java @@ -0,0 +1,108 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codenameone.developerguide.backend.security; + +import com.codename1.backend.DataSource; +import com.codename1.backend.annotations.Bean; +import com.codename1.backend.annotations.Configuration; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.security.Customizer; +import com.codename1.backend.security.HttpSecurity; +import com.codename1.backend.security.SecurityFilterChain; +import com.codename1.backend.security.core.userdetails.JdbcUserDetailsManager; +import com.codename1.backend.security.crypto.PasswordEncoder; +import com.codename1.backend.security.mail.EmailVerificationService; +import com.codename1.backend.security.mail.JdbcOneTimeTokenRepository; +import com.codename1.backend.security.mail.JdbcVerifiedEmailRepository; +import com.codename1.backend.security.mail.OneTimeTokenService; +import com.codename1.backend.security.mail.PasswordResetService; +import com.codename1.backend.security.mfa.EmailCodeService; +import com.codename1.backend.security.mfa.JdbcEmailCodeRepository; + +import java.io.IOException; + +/** The Backend security chapter's examples of the flows that send mail. */ +@Configuration +public class MailSecurityConfig { +// tag::backend-security-reset[] + @Bean + OneTimeTokenService oneTimeTokens(DataSource dataSource) { + return new OneTimeTokenService(new JdbcOneTimeTokenRepository(dataSource)); + } + + @Bean + PasswordResetService passwordReset(OneTimeTokenService tokens, MailSender mail, + JdbcUserDetailsManager users, PasswordEncoder encoder) { + return new PasswordResetService(tokens, mail, users, users, encoder); + } + + @Bean + SecurityFilterChain web(HttpSecurity http) { + http.authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) + .formLogin(Customizer.withDefaults()) + .passwordReset(Customizer.withDefaults()); + return http.build(); + } +// end::backend-security-reset[] + +// tag::backend-security-verify[] + @Bean + EmailVerificationService emailVerification(OneTimeTokenService tokens, MailSender mail, + DataSource dataSource) { + return new EmailVerificationService(tokens, mail, + new JdbcVerifiedEmailRepository(dataSource)); + } + + @Bean + SecurityFilterChain shop(HttpSecurity http, EmailVerificationService verification) { + http.authorizeHttpRequests(auth -> auth + .requestMatchers("/checkout/**").access(verification.verified()) + .anyRequest().authenticated()) + .formLogin(Customizer.withDefaults()) + .emailVerification(Customizer.withDefaults()); + return http.build(); + } +// end::backend-security-verify[] + +// tag::backend-security-verify-send[] + void afterSignup(EmailVerificationService verification, String username, String email) + throws IOException { + verification.send(username, email, "https://example.com/verify-email"); + } +// end::backend-security-verify-send[] + +// tag::backend-security-email-code[] + @Bean + EmailCodeService emailCodes(DataSource dataSource, MailSender mail) { + return new EmailCodeService(new JdbcEmailCodeRepository(dataSource), mail); + } + + @Bean + SecurityFilterChain account(HttpSecurity http) { + http.authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) + .formLogin(Customizer.withDefaults()) + .mfa(Customizer.withDefaults()); + return http.build(); + } +// end::backend-security-email-code[] +} diff --git a/docs/demos/backend/src/main/resources/application.properties b/docs/demos/backend/src/main/resources/application.properties index b66d768681b..0eb28219dc0 100644 --- a/docs/demos/backend/src/main/resources/application.properties +++ b/docs/demos/backend/src/main/resources/application.properties @@ -13,3 +13,8 @@ cn1.session.store=kv # ...and a broker for messages. cn1.messaging.url=kafka://kafka-1:9092,kafka-2:9092 # end::backend-scale-out[] + +# tag::backend-mail-config[] +cn1.mail.url=smtp://${SMTP_USER}:${SMTP_PASSWORD}@smtp.example.com:587 +cn1.mail.from=Acme +# end::backend-mail-config[] diff --git a/docs/demos/backend/src/main/snippets/mail-shipped.html b/docs/demos/backend/src/main/snippets/mail-shipped.html new file mode 100644 index 00000000000..c4e94892262 --- /dev/null +++ b/docs/demos/backend/src/main/snippets/mail-shipped.html @@ -0,0 +1,9 @@ + + + + +

Hello customer,

+

Order 1 is on its way.

+ + + diff --git a/docs/developer-guide/Backend-Mail.asciidoc b/docs/developer-guide/Backend-Mail.asciidoc new file mode 100644 index 00000000000..89dfc9192b1 --- /dev/null +++ b/docs/developer-guide/Backend-Mail.asciidoc @@ -0,0 +1,294 @@ +[[backend-mail]] +== Backend email + +Most servers send mail before they do much else: a welcome, a receipt, a link to +set a new password. This chapter covers sending it. One class describes a +message, one bean sends it, and the transport behind the bean is chosen by a +property. The transport is SMTP for a provider or a relay, the Amazon SES API, +or memory for development and tests. + +The names follow Spring where a Spring developer would look for them, and +Spring Boot's `spring.mail.*` properties are read. The surface is smaller. +`Mail` stands in for `SimpleMailMessage`, `MimeMessage` and +`MimeMessageHelper`, and `MailSender` for `JavaMailSender`. SMTP is implemented +on the backend's own network layer with no mail library, and the build links +only the transport the configuration names. A server that never injects +`MailSender` contains no mail code at all. + +=== Sending a mail + +Two settings are enough: + +[source,properties] +---- +include::../demos/backend/src/main/resources/application.properties[tag=backend-mail-config] +---- + +`cn1.mail.from` is the sender of every mail that doesn't name its own. Inject +`MailSender` like any other bean: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Signup.java[tag=backend-mail-send,indent=0] +---- + +`Mail.to(...)` starts a builder, and `build()` checks everything: an address +that isn't one, a header value with a line break in it, or an attachment with +no name throws `IllegalArgumentException` there. A `Mail` that exists can be +sent, so a malformed address fails in the code that produced it and not at the +mail server an hour later. Text built from user input is safe in any field, +because a value that would inject a header is refused and never escaped into +something else. + +`send` returns as soon as the mail is queued. Delivery happens on a thread of +the mail support's own, for two reasons. An SMTP exchange is several round +trips, and a request shouldn't wait for them. And a mail that fails the first +time has to be tried again after the request is long over. + +=== Sending inside a transaction + +When the server has a database, `send` writes the encoded mail to the table +`cn1_mail_outbox` in the caller's transaction. In the example above, the +account row and the welcome mail commit together. If `register` throws, the +transaction rolls back and no mail goes out. If the server stops right after +the commit, the row is still there, and the next server to start sends it. + +That's the transactional outbox <> uses for messages, and +it gives the same guarantee: a committed mail is sent at least once. In the +rare case where a server dies between sending the mail and removing the row, the mail +goes out twice. It never goes out zero times. + +Without a database the queue is in memory. Mail is retried the same way, and +whatever is still queued when the process stops is lost. + +=== Delivery and retries + +A mail the server couldn't take is tried again with a growing delay: 30 seconds, +then four times longer each time, up to an hour, for six attempts in all. What +happens next depends on why it failed: + +* A temporary failure, such as a `4xx` reply, a timeout or a dropped + connection, is retried. +* A permanent failure, a `5xx` reply, isn't. The server said it will never take + this mail, and sending it again only harms the sender's reputation. + +A mail that's given up on moves to `cn1_mail_dead` with the reason, and a line +in the log says so. Nothing sends from that table. It's there for an operator +to read. + +`sendNow` skips the queue. It delivers on the calling thread and throws +`MailException` when the server refuses, for the few cases where the caller has +to know before it answers: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java[tag=backend-mail-now,indent=0] +---- + +A mail sent with `sendNow` isn't part of the caller's transaction and isn't +retried. + +[cols="2,1,3", options="header"] +|=== +| Key | Default | What it sets + +| `cn1.mail.retry.max-attempts` +| 6 +| Attempts before a mail is given up on. + +| `cn1.mail.retry.backoff` +| 30000 +| Milliseconds before the second attempt. + +| `cn1.mail.retry.multiplier` +| 4 +| What each later wait is multiplied by. + +| `cn1.mail.retry.max-backoff` +| 3600000 +| The longest wait. + +| `cn1.mail.outbox.sweep` +| 10000 +| Milliseconds between looks for mails a stopped server left behind. +|=== + +=== Templates + +A mail's HTML can come from the same templates that render pages, described in +<>. Put the template under `templates/mail`: + +[source,html] +---- +include::../demos/backend/src/main/snippets/mail-shipped.html[tag=backend-mail-template-html,indent=0] +---- + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java[tag=backend-mail-template,indent=0] +---- + +The template is compiled by the build like every other, so a property the +model doesn't have fails the build with the file and position, and nothing is +parsed when a mail is sent. + +Every HTML mail should carry a plain text version too. Mail clients that show +text need it, and spam filters score a mail without one. The text is derived +from the rendered HTML, with the tags removed and the address of each link kept +beside its words. Calling `text(...)` on the builder gives your own instead. + +A mail is read outside your site. A template for one can't use a relative link +or a stylesheet, and has no request to read. Write absolute URLs and inline +styles. + +=== Attachments and inline images + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/Receipts.java[tag=backend-mail-attach,indent=0] +---- + +`attach` adds a file the reader can save. `inline` adds an image the HTML +shows in place, referenced as `cid:` followed by the content id. The builder +picks the structure a mail client expects from what the mail contains, so +there's no multipart mode to choose. + +Subjects, display names and file names can hold any Unicode text. Bodies are +always UTF-8. `header` sets a header of your own, such as `List-Unsubscribe`. + +=== Providers + +[cols="2,3", options="header"] +|=== +| `cn1.mail.url` | Sends through + +| `smtp://user:password@host:587` +| SMTP with STARTTLS. Every provider with an SMTP endpoint, and any relay. + +| `smtps://user:password@host:465` +| SMTP over TLS from the first byte. + +| `ses://eu-west-1` +| The Amazon SES API in that region. + +| `log:` +| Nothing. Each mail's headers and text are written to the log. + +| `memory:` +| Nothing. Mails are kept for a test to read. The default. +|=== + +Without `cn1.mail.url`, Spring Boot's `spring.mail.host`, `spring.mail.port`, +`spring.mail.username` and `spring.mail.password` select SMTP. + +==== SMTP + +A username or password containing `@`, `:` or `/` has to be URL-encoded in the +URL, or set apart from it with `cn1.mail.username` and `cn1.mail.password`. + +`smtp://` upgrades the connection with STARTTLS before it says anything else, +and refuses a server that doesn't offer it. A password is never sent over a +connection that isn't encrypted, whatever the settings say. For a relay on the +same machine that has no TLS, `cn1.mail.smtp.starttls=false` sends in the +clear, without credentials. + +Connections are kept open between mails and reused, and several mails handed to +`send(Mail...)` in one call go over one connection. + +[cols="2,1,3", options="header"] +|=== +| Key | Default | What it sets + +| `cn1.mail.smtp.auth` +| +| `plain`, `login` or `xoauth2`. Unset takes what the server offers. With + `xoauth2` the password is an OAuth 2 access token. + +| `cn1.mail.smtp.helo` +| `localhost` +| The name the client gives for itself. + +| `cn1.mail.smtp.ca` +| +| A PEM file of certificates to trust in place of the system's. + +| `cn1.mail.timeout.connect` +| 5000 +| Milliseconds a connection may take to open. + +| `cn1.mail.timeout.read` +| 60000 +| Milliseconds a reply may take. + +| `cn1.mail.pool.max` +| 2 +| Idle connections kept for the next mail. + +| `cn1.mail.pool.idle` +| 30000 +| Milliseconds an idle connection is kept. +|=== + +==== Amazon SES + +`ses://` followed by a region sends through the SES API: one signed HTTPS +request for each mail, with no SMTP credentials to issue. The credentials are +the machine's own, read from the environment, a container role or an instance +role, as for the other AWS services in <>. The sender has to +be an identity SES has verified. `cn1.mail.ses.configuration-set` names a +configuration set. + +=== Testing + +A test that names no `cn1.mail.url` sends nothing. Mails are kept in memory, and +`MailSender` hands them back: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/mail/SignupTest.java[tag=backend-mail-test,indent=0] +---- + +`sent()` returns the mails in the order they were sent, with templates already +rendered, so `getHtml()` and `getText()` are what a reader would have seen. +Mail is captured during the call to `send`, so there's nothing to wait for. + +During development `cn1.mail.url=log:` prints each mail. To look at mails in a +real client, run a catch-all SMTP server on your machine and point +`cn1.mail.url` at `smtp://localhost:1025` with `cn1.mail.smtp.starttls=false`. + +=== Observing mail + +The counters `cn1.mail.sent`, `cn1.mail.failed`, `cn1.mail.refused` and +`cn1.mail.retried`, and the gauge `cn1.mail.outbox.pending`, are exported with +the metrics <> describes. Every hand-over to the +mail server is a span named `mail send`, with the transport, the number of +recipients and the size of the message. No address is ever written to a span, a +metric or the log. + +Once a server has sent a mail, its health endpoint reports the transport under +`mail`. When the transport is down, that's reported and the server stays in +service: mail waits in the outbox, and the server's requests are no worse +served for it. + +=== What isn't supported + +* Receiving mail. There's no IMAP or POP3 client. +* Signing with DKIM. The provider or the relay signs. +* S/MIME and PGP. +* Character sets other than UTF-8. + +=== Pitfalls + +* *No `cn1.mail.url` in production.* The default keeps mail in memory and + sends nothing. The server says so in its log the first time a mail is sent. +* *A sender the provider hasn't verified.* Providers refuse mail from a domain + that isn't yours to send from. That's a permanent failure, so every mail + lands in `cn1_mail_dead`. +* *Waiting on `sendNow` in a request.* It holds the request for the whole SMTP + exchange. Use `send` unless the answer depends on the outcome. +* *Treating accepted as delivered.* The server accepting a mail says nothing + about the inbox. Bounces and complaints come back through the provider's + webhooks or its bounce address. +* *A handler that isn't safe to repeat.* A committed mail can go out twice + after a crash. Put what makes a mail unique, such as an order number, where + the reader sees it. diff --git a/docs/developer-guide/Backend-Security.asciidoc b/docs/developer-guide/Backend-Security.asciidoc index 35da9e64611..91c07b30153 100644 --- a/docs/developer-guide/Backend-Security.asciidoc +++ b/docs/developer-guide/Backend-Security.asciidoc @@ -1340,6 +1340,41 @@ Codes are six digits that change every 30 seconds, which is what authenticator apps expect. One step either side of the current one is accepted, to allow for a clock that's off. A code is good once. +[[backend-security-email-code]] +==== A code by mail + +A user without an authenticator app can be mailed a code. Declare an +`EmailCodeService` beside the `TotpService`, or in its place: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java[tag=backend-security-email-code,indent=0] +---- + +`emailCodes.enable(username)` turns it on for a user, and `disable` turns it +off. A user with only this factor is mailed a code as soon as their password +is accepted. A user who also has an authenticator app is asked for the app's +code, and the page has a button that mails one. Either is accepted. + +A mailed code is six digits and good for ten minutes, once. Five wrong guesses +spend it, whoever makes them. A new code can be asked for every 30 seconds and +replaces the one before it, but it doesn't bring five more guesses: the count +carries over until a code is entered correctly or the last one runs out. Once +the five are spent, no code is mailed until then. The server stores a +hash of the code, bound to the user it was made for. `validitySeconds`, +`maxAttempts` and `resendSeconds` change the numbers, and `template` renders +the mail from a template of your own, which is given `code`, `username` and +`minutes`. + +The code goes to the address `EmailAddressResolver` gives for the user. The +default takes a username that's an address as it is. An +`EmailVerificationService` is such a resolver too, and passing it to +`addressResolver` sends codes only to addresses users have proved are theirs. + +A mailed code is as strong as the user's mailbox. It stops a stolen password +from being enough. It doesn't stop someone who reads the user's mail, which an +authenticator app or a passkey does. + ==== Recovery codes `RecoveryCodeService.generate` makes ten codes that stand in for a lost phone. @@ -1486,6 +1521,91 @@ holds a secret: a public key verifies and can't sign. A user who signs in with a passkey is looked up in the application's `UserDetailsService` for their authorities. +[[backend-security-password-reset]] +=== Resetting a forgotten password + +`passwordReset` adds the two pages a forgotten password needs, and mails the +link between them through the `MailSender` of <>: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java[tag=backend-security-reset,indent=0] +---- + +`GET /forgot-password` asks for the username. Posting it mails a link to +`/reset-password`, where the user chooses a new password. The link works once +and for 30 minutes. A newer link replaces an older one, and after a reset the +user's remembered sign-ins are removed, so a browser that was remembered has +to sign in again. A session that's already signed in stays until it ends. +`requestUrl` and `resetUrl` move the pages, and `servePages(false)` leaves the +`GET` of both to controllers of your own, which post to the same paths with +the fields `username`, and `token`, `password` and `confirm`. + +The login page the layer generates gains a "Forgot your password?" link to the +request page, and tells a user the reset sends back that the password was +changed. A login page of your own links there itself; the reset returns to it +with `?reset` in the address. + +What the layer does so that the pages can't be turned against your users: + +* *The answer never says whether an account exists.* A username nobody has + gets the same redirect, and no mail. An account that's disabled or has no + address is answered the same way. +* *Asking is rate limited.* Five requests in fifteen minutes from one network, + and five for one username, then `429 Too Many Requests`. A `RateLimiter` + bean is used when there is one, so the limit holds across instances. See + <>. +* *Only a hash of the token is stored.* A copy of the database can't be used to + reset anyone's password. +* *The link is never built from the request's `Host` header.* That header is + whatever the client sent, and a link built from it would mail your user a + link to someone else's site. Set `cn1.security.base-url=https://example.com`, + or call `baseUrl(...)`. Without it only a request to `localhost` is served, + which is enough for development, and any other fails naming the setting. +* *The pages aren't cached and send no referrer,* so the token in the address + doesn't leave the browser. + +The new password has to be eight characters or more. `minimumLength` on the +service changes that, `validityMinutes` the life of a link, and `template` +names a mail template of your own, which is given `link`, `username` and +`minutes`. The mail goes to the address `EmailAddressResolver` gives for the +user, as in <>. + +[[backend-security-email-verification]] +=== Verifying an email address + +An address a user typed at sign-up is a claim. `EmailVerificationService` +turns it into a fact by mailing a link to it: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java[tag=backend-security-verify,indent=0] +---- + +Send the mail when the account is created, or when the user changes their +address: + +[source,java,role=cn1-no-try] +---- +include::../demos/backend/src/main/java/com/codenameone/developerguide/backend/security/MailSecurityConfig.java[tag=backend-security-verify-send,indent=0] +---- + +`emailVerification` serves `/verify-email`, the page the link opens. It has +one button. Pressing it marks the address as verified and redirects to +`/?verified`, or to `successUrl`. Opening the link isn't enough by itself, +because mail gateways open every link in a message to scan it, before the +reader sees the message. The user doesn't have to be signed in when they +follow the link. Most people open mail on a different device from the one +they signed up on. A link works once and for a day, and a link that's spent +or made up gets a `400` page. A link proves only the address it was mailed +to. + +`verification.verified()` is an authorization rule: the request needs a +signed-in user whose address is verified. `isVerified(username)` answers the +same question in code. Sending again for a user replaces their pending address +and their earlier link. An address stays verified until `send` is called with +another or `forget` removes it. + === Remembering a user `rememberMe` gives a user who ticks the box on the login form a cookie that @@ -1622,6 +1742,10 @@ are the same on SQLite, PostgreSQL, and MySQL. | 12 | `cn1_persistent_logins.user_agent`, `cn1_persistent_logins.ip_address` | The device each remembered sign-in came from. + +| 13 +| `cn1_one_time_token`, `cn1_mfa_email`, `cn1_verified_email` +| `JdbcOneTimeTokenRepository`, `JdbcEmailCodeRepository`, `JdbcVerifiedEmailRepository` |=== To use tables you already have, give their names in `application.properties`. @@ -1977,6 +2101,11 @@ read by the build, because it decides what's linked into the server. password is set. The name is `user` unless set, and the roles are separated by commas. A password without a `{id}` prefix is taken as `{noop}`. +| `cn1.security.base-url` +| The address the server is reached at, such as `https://example.com`, with + nothing after the host. Mailed links start with it. See + <>. + | `cn1.security.password.maxConcurrent` | The most passwords checked at one time. No bound unless set. diff --git a/docs/developer-guide/Backend-Testing.asciidoc b/docs/developer-guide/Backend-Testing.asciidoc index 7eb4ae2c285..cd13350e459 100644 --- a/docs/developer-guide/Backend-Testing.asciidoc +++ b/docs/developer-guide/Backend-Testing.asciidoc @@ -278,6 +278,15 @@ interface the application never implements and gets the generated one. Both slices take `properties`, `profile` and `classes` as `@BackendTest` does, and both run compiled, as the next sections describe. +=== Mail in a test + +A test's application sends no mail unless the test names a `cn1.mail.url`. +Every mail is kept in memory instead, and `MailSender.sent()` returns them, so +a test can assert on the recipient, the subject and the rendered body. The +flows that mail a link or a code, such as a password reset, are tested the same +way: read the link out of the captured mail and follow it. <> has +an example. + === Turning production features on A test's application starts without the management endpoints, the MCP endpoint diff --git a/docs/developer-guide/developer-guide.asciidoc b/docs/developer-guide/developer-guide.asciidoc index a0601ed86f3..6cd2acdd726 100644 --- a/docs/developer-guide/developer-guide.asciidoc +++ b/docs/developer-guide/developer-guide.asciidoc @@ -143,6 +143,8 @@ include::Backend-Scheduling.asciidoc[] include::Backend-Messaging.asciidoc[] +include::Backend-Mail.asciidoc[] + include::Backend-Observability.asciidoc[] include::Backend-MCP.asciidoc[] diff --git a/maven/backend/src/test/java/com/codename1/backend/security/MailFlowsTest.java b/maven/backend/src/test/java/com/codename1/backend/security/MailFlowsTest.java new file mode 100644 index 00000000000..ecc66343c0d --- /dev/null +++ b/maven/backend/src/test/java/com/codename1/backend/security/MailFlowsTest.java @@ -0,0 +1,492 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertNotNull; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +import com.codename1.backend.HttpServer; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.mvc.Model; +import com.codename1.backend.security.SecuredServer.Reply; +import com.codename1.backend.security.crypto.PasswordEncoderFactories; +import com.codename1.backend.security.core.userdetails.InMemoryUserDetailsManager; +import com.codename1.backend.security.core.userdetails.User; +import com.codename1.backend.security.mail.EmailVerificationService; +import com.codename1.backend.security.mail.InMemoryOneTimeTokenRepository; +import com.codename1.backend.security.mail.InMemoryVerifiedEmailRepository; +import com.codename1.backend.security.mail.OneTimeTokenService; +import com.codename1.backend.security.mail.PasswordResetService; +import com.codename1.backend.security.mfa.EmailCodeService; +import com.codename1.backend.security.mfa.InMemoryEmailCodeRepository; +import com.codename1.backend.security.mfa.InMemoryRecoveryCodeRepository; +import com.codename1.backend.security.mfa.RecoveryCodeService; +import com.codename1.impl.backend.MailRuntime; +import com.codename1.impl.backend.security.SecuritySupport; +import java.io.IOException; +import java.util.List; +import java.util.Properties; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +/// The flows that end in an inbox, through a real chain: the forgotten +/// password pages, the link that proves an address, and the sign-in code that +/// is mailed. Mail lands in the memory transport, which is what a server with +/// no `cn1.mail.url` sends through. +class MailFlowsTest { + private static final String ADA = "ada@example.com"; + + private static final HttpServer.Handler APP = new HttpServer.Handler() { + @Override + public HttpServer.Response handle(HttpServer.Request request) { + Authentication who = SecuritySupport.authentication(); + return HttpServer.Response.text(200, request.pathFrom(0) + " " + + (who == null ? "nobody" : who.getName())); + } + }; + + private static final class Ticking implements Clock { + long now = 1700000000000L; + + @Override + public long currentTimeMillis() { + return now; + } + } + + private final Ticking clock = new Ticking(); + private final MailSender mail = new MailSender(); + private final OneTimeTokenService tokens = new OneTimeTokenService( + new InMemoryOneTimeTokenRepository()).clock(clock); + private final InMemoryUserDetailsManager users = new InMemoryUserDetailsManager( + User.withUsername(ADA).password("{noop}old password").roles("USER").build(), + User.withUsername("ray").password("{noop}ray password").roles("USER").build()); + + private static Properties settings() { + Properties settings = SecuredServer.settings(); + settings.setProperty("cn1.mail.from", "Accounts "); + // The code's own count of guesses is what these tests are about, so + // the sign-in's rate limit is moved out of its way. + settings.setProperty(MfaConfigurer.ATTEMPTS, "50"); + settings.setProperty(MfaConfigurer.ATTEMPTS_PER_ADDRESS, "50"); + return settings; + } + + /// The one link in the newest mail. + private String link(String to) throws Exception { + List sent = mail.sent(); + assertFalse(sent.isEmpty(), "nothing was mailed"); + Mail last = sent.get(sent.size() - 1); + assertEquals(to, last.getTo().get(0)); + String text = last.getText(); + int at = text.indexOf("http"); + assertTrue(at >= 0, text); + int end = at; + while (end < text.length() && text.charAt(end) > ' ') { + end++; + } + return text.substring(at, end); + } + + /// The path and query of a link, which is what a request is made of. + private static String target(String link) { + return link.substring(link.indexOf('/', link.indexOf("//") + 2)); + } + + // -------------------------------------------------------- password reset + + private SecuredServer startReset(Properties settings, String baseUrl) throws Exception { + PasswordResetService service = new PasswordResetService(tokens, mail, users, users, + PasswordEncoderFactories.createDelegatingPasswordEncoder()); + return SecuredServer.start(settings, "dev", new Object[] {users}, APP, + http -> http.authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) + .csrf(csrf -> csrf.disable()) + .formLogin(Customizer.withDefaults()) + .passwordReset(reset -> { + reset.service(service); + if (baseUrl != null) { + reset.baseUrl(baseUrl); + } + }).build()); + } + + @Test + @DisplayName("a forgotten password is replaced through a mailed link, once") + void passwordReset() throws Exception { + try (SecuredServer server = startReset(settings(), null)) { + mail.clearSent(); + // The generated login page offers the way there, and says what + // happened to someone a reset sends back. + String login = server.get("/login").body; + assertTrue(login.contains("Forgot your password?"), + login); + assertFalse(login.contains("Your password was changed"), login); + assertTrue(server.get("/login?reset").body.contains("Your password was changed")); + + // Both pages are served to nobody. + Reply ask = server.get("/forgot-password"); + assertEquals(200, ask.status); + assertTrue(ask.body.contains("name=\"username\""), ask.body); + assertEquals("no-store", ask.header("Cache-Control")); + + // An account that does not exist is answered as one that does. + Reply unknown = server.post("/forgot-password", "username=nobody%40example.com"); + assertEquals("302 /forgot-password?sent", + unknown.status + " " + unknown.header("Location")); + assertEquals(0, mail.sent().size()); + Reply known = server.post("/forgot-password", "username=ada%40example.com"); + assertEquals("302 /forgot-password?sent", + known.status + " " + known.header("Location")); + assertEquals(1, mail.sent().size()); + assertEquals(server.get("/forgot-password?sent").body.length() > 0, true); + + // The request named localhost, so the link may. + String link = link(ADA); + assertTrue(link.startsWith("http://localhost/reset-password?token="), link); + Reply form = server.get(target(link)); + assertEquals(200, form.status); + assertTrue(form.body.contains("name=\"password\"") + && form.body.contains("name=\"confirm\""), form.body); + assertEquals("no-referrer", form.header("Referrer-Policy")); + String token = link.substring(link.indexOf("token=") + 6); + + // A password that is refused keeps the link alive. + Reply shortOne = server.post("/reset-password", + "token=" + token + "&password=abc&confirm=abc"); + assertEquals("302 /reset-password?token=" + token + "&invalid", + shortOne.status + " " + shortOne.header("Location")); + Reply differ = server.post("/reset-password", + "token=" + token + "&password=new+password&confirm=another+one"); + assertEquals("302 /reset-password?token=" + token + "&invalid", + differ.status + " " + differ.header("Location")); + + Reply done = server.post("/reset-password", + "token=" + token + "&password=new+password&confirm=new+password"); + assertEquals("302 /login?reset", done.status + " " + done.header("Location")); + // Resetting a password signs nobody in. + assertEquals("/login", server.get("/me", "Accept", "text/html").header("Location")); + + // The link is spent. + Reply again = server.post("/reset-password", + "token=" + token + "&password=third+password&confirm=third+password"); + assertEquals("302 /forgot-password?expired", + again.status + " " + again.header("Location")); + + assertEquals("/login?error", server.post("/login", + "username=ada%40example.com&password=old+password").header("Location")); + // Back to the page that was asked for before the sign-in. + assertEquals("/me", server.post("/login", + "username=ada%40example.com&password=new+password").header("Location")); + assertEquals("/me " + ADA, server.get("/me").body); + + // What the health report and the outbox gauge read. + MailRuntime.Probe probe = MailRuntime.probe(); + assertNotNull(probe); + assertEquals("memory", probe.transport()); + assertTrue(probe.isHealthy()); + assertEquals(0, probe.pending()); + } + } + + @Test + @DisplayName("a reset link dies with its half hour, and a made-up one is refused") + void resetLinksExpire() throws Exception { + try (SecuredServer server = startReset(settings(), null)) { + mail.clearSent(); + server.post("/forgot-password", "username=ada%40example.com"); + String token = link(ADA).substring(link(ADA).indexOf("token=") + 6); + clock.now += 31 * 60000L; + Reply late = server.post("/reset-password", + "token=" + token + "&password=new+password&confirm=new+password"); + assertEquals("302 /forgot-password?expired", + late.status + " " + late.header("Location")); + Reply none = server.get("/reset-password"); + assertEquals("302 /forgot-password?expired", + none.status + " " + none.header("Location")); + Reply invented = server.post("/reset-password", + "token=AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA" + + "&password=new+password&confirm=new+password"); + assertEquals("302 /forgot-password?expired", + invented.status + " " + invented.header("Location")); + assertEquals("/", server.post("/login", + "username=ada%40example.com&password=old+password").header("Location")); + } + } + + @Test + @DisplayName("asking for links is rate limited, by network and by account") + void resetRequestsAreLimited() throws Exception { + try (SecuredServer server = startReset(settings(), null)) { + mail.clearSent(); + for (int iter = 0 ; iter < 5 ; iter++) { + assertEquals(302, server.post("/forgot-password", + "username=ada%40example.com").status); + } + Reply refused = server.post("/forgot-password", "username=ada%40example.com"); + assertEquals(429, refused.status, refused.toString()); + assertNotNull(refused.header("Retry-After"), refused.toString()); + assertEquals(5, mail.sent().size()); + } + } + + @Test + @DisplayName("a link is never built from the Host header of a request") + void theLinkNamesTheConfiguredAddress() throws Exception { + // Configured in code. + try (SecuredServer server = startReset(settings(), "https://shop.example.com/")) { + mail.clearSent(); + server.post("/forgot-password", "username=ada%40example.com", + "Host", "attacker.example"); + assertTrue(link(ADA).startsWith("https://shop.example.com/reset-password?token="), + link(ADA)); + } + // Configured in the properties. + Properties settings = settings(); + settings.setProperty(PasswordResetConfigurer.BASE_URL, "https://app.example.com"); + try (SecuredServer server = startReset(settings, null)) { + mail.clearSent(); + server.post("/forgot-password", "username=ada%40example.com", + "Host", "attacker.example"); + assertTrue(link(ADA).startsWith("https://app.example.com/reset-password?token="), + link(ADA)); + } + // Not configured: localhost is served, and anything else fails + // naming the setting, before any account is looked at. + try (SecuredServer server = startReset(settings(), null)) { + mail.clearSent(); + IllegalStateException refused = assertThrows(IllegalStateException.class, + () -> server.post("/forgot-password", "username=ada%40example.com", + "Host", "attacker.example")); + assertTrue(refused.getMessage().contains(PasswordResetConfigurer.BASE_URL), + refused.getMessage()); + assertEquals(0, mail.sent().size()); + } + assertThrows(Exception.class, + () -> startReset(settings(), "https://shop.example.com/app").close()); + } + + @Test + @DisplayName("a mail's template is rendered when it is sent, and its text derived from it") + void templatesRenderAtSendTime() throws Exception { + try (SecuredServer server = startReset(settings(), null)) { + mail.clearSent(); + // No templates in this application: the mail says where one goes. + MailRuntime.install(null); + IOException none = assertThrows(IOException.class, () -> mail.send( + Mail.to(ADA).subject("Hello").template("mail/welcome", new Model()).build())); + assertTrue(none.getMessage().contains("templates/mail/welcome.html"), + none.getMessage()); + + // What the generated wiring installs over the compiled templates. + MailRuntime.install(new MailRuntime.TemplateRenderer() { + @Override + public byte[] render(String view, Model model) throws IOException { + return ("

" + view + " for " + + model.getAttribute("name") + "

").getBytes("UTF-8"); + } + }); + try { + mail.send(Mail.to(ADA).subject("Hello") + .template("mail/welcome", new Model().addAttribute("name", "Ada")) + .build()); + Mail sent = mail.sent().get(0); + assertEquals("

mail/welcome for Ada

", + sent.getHtml()); + assertTrue(sent.getText().contains("mail/welcome for Ada") + && sent.getText().contains("https://example.com/a"), sent.getText()); + assertEquals("Accounts ", sent.getFrom()); + + // Text of the caller's own is kept. + mail.send(Mail.to(ADA).subject("Hello").text("Plain words") + .template("mail/welcome", new Model().addAttribute("name", "Ada")) + .build()); + assertEquals("Plain words", mail.sent().get(1).getText()); + } finally { + MailRuntime.install(null); + } + } + } + + // ---------------------------------------------------- email verification + + @Test + @DisplayName("following the mailed link proves the address, and opens what asks for it") + void emailVerification() throws Exception { + EmailVerificationService service = new EmailVerificationService(tokens, mail, + new InMemoryVerifiedEmailRepository()).clock(clock); + try (SecuredServer server = SecuredServer.start(settings(), "dev", new Object[] {users}, + APP, http -> http.authorizeHttpRequests(auth -> auth + .requestMatchers("/orders").access(service.verified()) + .anyRequest().authenticated()) + .csrf(csrf -> csrf.disable()) + .formLogin(Customizer.withDefaults()) + .emailVerification(verify -> verify.service(service)).build())) { + mail.clearSent(); + // No reset in this chain: its login page offers none, and has + // nothing to say to a ?reset somebody typed. + String login = server.get("/login?reset").body; + assertFalse(login.contains("Forgot your password") + || login.contains("Your password was changed"), login); + server.post("/login", "username=ray&password=ray+password"); + assertEquals("/me ray", server.get("/me").body); + assertEquals(403, server.get("/orders").status); + + service.send("ray", "ray@example.com", "http://localhost/verify-email"); + String link = link("ray@example.com"); + assertFalse(service.isVerified("ray")); + + // A link that was never mailed. + assertEquals(400, server.get("/verify-email?token=nothing").status); + assertEquals(400, server.get("/verify-email").status); + assertFalse(service.isVerified("ray")); + + // The real one, followed in a browser that is not signed in. + String session = server.cookies.remove("CN1SESSION"); + // Opening it proves nothing: a mail gateway opens every link. + Reply page = server.get(target(link)); + String token = link.substring(link.indexOf("token=") + 6); + assertEquals(200, page.status); + assertTrue(page.body.contains("
") + && page.body.contains("name=\"token\" value=\"" + token + "\""), page.body); + assertEquals("no-store", page.header("Cache-Control")); + assertFalse(service.isVerified("ray")); + // Pressing the button does. + Reply done = server.post("/verify-email", "token=" + token); + assertEquals("302 /?verified", done.status + " " + done.header("Location")); + assertTrue(service.isVerified("ray")); + assertEquals("ray@example.com", service.getVerifiedEmail("ray")); + assertEquals(400, server.post("/verify-email", "token=" + token).status); + assertEquals(400, server.post("/verify-email", "token=%22%3E%3Cscript%3E").status); + + server.cookies.put("CN1SESSION", session); + assertEquals("/orders ray", server.get("/orders").body); + } + } + + // ------------------------------------------------------- the mailed code + + private String mailedCode() throws Exception { + List sent = mail.sent(); + assertFalse(sent.isEmpty(), "nothing was mailed"); + String text = sent.get(sent.size() - 1).getText(); + String lead = "Your sign-in code is "; + int at = text.indexOf(lead); + assertTrue(at >= 0, text); + return text.substring(at + lead.length(), at + lead.length() + 6); + } + + @Test + @DisplayName("a user whose second factor is a mailed code is sent one and signs in with it") + void mailedCodeSignIn() throws Exception { + EmailCodeService codes = new EmailCodeService(new InMemoryEmailCodeRepository(), mail) + .clock(clock); + codes.enable(ADA); + RecoveryCodeService recovery = new RecoveryCodeService( + new InMemoryRecoveryCodeRepository()); + try (SecuredServer server = SecuredServer.start(settings(), "dev", + new Object[] {users, codes, recovery}, APP, + http -> http.authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) + .csrf(csrf -> csrf.disable()) + .formLogin(Customizer.withDefaults()) + .mfa(mfa -> mfa.clock(clock)).build())) { + mail.clearSent(); + // No second factor: straight in. + assertEquals("/", server.post("/login", "username=ray&password=ray+password") + .header("Location")); + server.cookies.clear(); + assertEquals(0, mail.sent().size()); + + Reply first = server.post("/login", + "username=ada%40example.com&password=old+password"); + assertEquals("302 /login/mfa", first.status + " " + first.header("Location")); + // The password was enough to be mailed a code, and for nothing else. + assertEquals(1, mail.sent().size()); + assertEquals("/login", server.get("/me", "Accept", "text/html").header("Location")); + Reply page = server.get("/login/mfa"); + assertTrue(page.body.contains("name=\"code\"") && page.body.contains("name=\"send\""), + page.body); + + String code = mailedCode(); + String wrong = "000000".equals(code) ? "000001" : "000000"; + Reply refused = server.post("/login/mfa", "code=" + wrong); + assertEquals("302 /login/mfa?error", + refused.status + " " + refused.header("Location")); + + // Another one too soon is not mailed; after the pause it is, and + // it replaces the first. + Reply soon = server.post("/login/mfa", "send=1"); + assertEquals("302 /login/mfa?sent", soon.status + " " + soon.header("Location")); + assertEquals(1, mail.sent().size()); + clock.now += 31000; + server.post("/login/mfa", "send=1"); + assertEquals(2, mail.sent().size()); + String second = mailedCode(); + if (!second.equals(code)) { + assertEquals("/login/mfa?error", + server.post("/login/mfa", "code=" + code).header("Location")); + } + Reply done = server.post("/login/mfa", "code=" + second); + assertEquals("302 /me", done.status + " " + done.header("Location")); + assertEquals("/me " + ADA, server.get("/me").body); + // Spent with the sign-in it made. + assertFalse(codes.verify(ADA, second)); + } + } + + @Test + @DisplayName("a mailed code runs out, and guesses run out before it does") + void mailedCodesAreBounded() throws Exception { + EmailCodeService codes = new EmailCodeService(new InMemoryEmailCodeRepository(), mail) + .clock(clock); + codes.enable(ADA); + try (SecuredServer server = SecuredServer.start(settings(), "dev", + new Object[] {users, codes}, APP, + http -> http.authorizeHttpRequests(auth -> auth.anyRequest().authenticated()) + .csrf(csrf -> csrf.disable()) + .formLogin(Customizer.withDefaults()) + .mfa(mfa -> mfa.clock(clock)).build())) { + mail.clearSent(); + server.post("/login", "username=ada%40example.com&password=old+password"); + String code = mailedCode(); + String wrong = "000000".equals(code) ? "000001" : "000000"; + for (int iter = 0 ; iter < 5 ; iter++) { + server.post("/login/mfa", "code=" + wrong); + } + // Five wrong guesses and the right one is worth nothing. + assertEquals("/login/mfa?error", + server.post("/login/mfa", "code=" + code).header("Location")); + + clock.now += 31000; + server.post("/login/mfa", "send=1"); + String late = mailedCode(); + clock.now += 601000; + Reply expired = server.post("/login/mfa", "code=" + late); + assertFalse("/".equals(expired.header("Location")), expired.toString()); + assertEquals("/login", server.get("/me", "Accept", "text/html").header("Location")); + } + } +} diff --git a/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java b/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java index f8c65c3d319..0e9de9afcf7 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/SecuritySchemaHistoryTest.java @@ -95,7 +95,7 @@ void upgradingRememberMeKeepsExistingTokens() throws Exception { Migrations.of(pool, SecuritySchema.migrations()).target("10").migrate(); pool.execute("INSERT INTO cn1_persistent_logins (series, username_key, username, token_hash, last_used) " + "VALUES ('browser', 'ada', 'Ada', 'old-hash', 1000)", null); - assertEquals(2, Migrations.of(pool, SecuritySchema.migrations()).migrate().getMigrationsExecuted()); + assertEquals(3, Migrations.of(pool, SecuritySchema.migrations()).migrate().getMigrationsExecuted()); com.codename1.backend.security.rememberme.JdbcTokenRepository repository = new com.codename1.backend.security.rememberme.JdbcTokenRepository(pool); assertEquals("old-hash", repository.getTokenForSeries("browser").getTokenHash()); @@ -113,9 +113,9 @@ void everyVersionHasItsOwnName() throws Exception { DataSource pool = DataSource.open(new File(dir, "named.db").getPath(), 2, 5000, 10000); try { MigrateResult result = Migrations.of(pool, SecuritySchema.migrations()).migrate(); - assertEquals(12, result.getMigrationsExecuted()); - assertEquals(12, new TreeSet(result.getApplied()).size(), - "twelve versions reported under fewer names: " + result.getApplied()); + assertEquals(13, result.getMigrationsExecuted()); + assertEquals(13, new TreeSet(result.getApplied()).size(), + "thirteen versions reported under fewer names: " + result.getApplied()); assertEquals("com.codename1.backend.security.SecuritySchema.V1__users_and_authorities", result.getApplied().get(0)); assertEquals("com.codename1.backend.security.SecuritySchema.V9__passkeys", @@ -140,9 +140,9 @@ void aHistoryWrittenBeforeTheNamesStillValidates() throws Exception { Migrations.of(pool, SecuritySchema.migrations()).validate(); MigrateResult again = Migrations.of(pool, SecuritySchema.migrations()).migrate(); assertEquals(0, again.getMigrationsExecuted()); - assertEquals("12", again.getTargetVersion()); + assertEquals("13", again.getTargetVersion()); MigrationInfo[] info = Migrations.of(pool, SecuritySchema.migrations()).info(); - assertEquals(12, info.length); + assertEquals(13, info.length); for (MigrationInfo row : info) { assertEquals(MigrationState.SUCCESS, row.getState(), "version " + row.getVersion()); } @@ -161,7 +161,7 @@ void rememberMeRecordsAndListsTheDeviceOnAnUpgradedTable() throws Exception { Migrations.of(pool, SecuritySchema.migrations()).target("11").migrate(); pool.execute("INSERT INTO cn1_persistent_logins (series, username_key, username, " + "token_hash, last_used) VALUES ('old', 'ada', 'Ada', 'h0', 1000)", null); - assertEquals(1, Migrations.of(pool, SecuritySchema.migrations()).migrate() + assertEquals(2, Migrations.of(pool, SecuritySchema.migrations()).migrate() .getMigrationsExecuted()); com.codename1.backend.security.rememberme.JdbcTokenRepository repository = new com.codename1.backend.security.rememberme.JdbcTokenRepository(pool); diff --git a/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java b/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java index 7baadb47f7f..9cbf2818593 100644 --- a/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java +++ b/maven/backend/src/test/java/com/codename1/backend/security/SecurityStoresTest.java @@ -84,6 +84,7 @@ class SecurityStoresTest { "cn1_persistent_logins", "cn1_mfa_totp", "cn1_mfa_recovery_code", "cn1_rate_limit", "cn1_federated_identity", "cn1_oauth2_registered_client", "cn1_oauth2_authorization", "cn1_oauth2_token", "cn1_webauthn_user", "cn1_webauthn_credential", + "cn1_one_time_token", "cn1_mfa_email", "cn1_verified_email", "cn1_security_schema_history"}; private final List shared = new ArrayList(); @@ -148,7 +149,7 @@ void theSchemaIsAppliedOnceUnderItsOwnHistory(String engine) throws Exception { } } MigrationInfo[] info = Migrations.of(pool, SecuritySchema.migrations()).info(); - assertEquals(12, info.length); + assertEquals(13, info.length); for (int i = 0; i < info.length; i++) { assertEquals(String.valueOf(i + 1), info[i].getVersion()); assertEquals(MigrationState.SUCCESS, info[i].getState()); @@ -161,12 +162,12 @@ void theSchemaIsAppliedOnceUnderItsOwnHistory(String engine) throws Exception { assertEquals("passkeys", info[8].getDescription()); assertEquals("rate limit expiry", info[9].getDescription()); // Its own history, not the application's. - assertEquals(12L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM " + assertEquals(13L, ((Number) pool.queryOne("SELECT COUNT(*) AS n FROM " + "cn1_security_schema_history", null).get("n")).longValue()); // And a second run finds nothing to do. MigrateResult again = Migrations.of(pool, SecuritySchema.migrations()).migrate(); assertEquals(0, again.getMigrationsExecuted()); - assertEquals("12", again.getTargetVersion()); + assertEquals("13", again.getTargetVersion()); assertEquals("security", SecuritySchema.migrations().getName()); assertEquals("cn1_security_schema_history", SecuritySchema.migrations().getTable()); } diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendBeans.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendBeans.java index aabb61deb46..4f28e289cbb 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendBeans.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendBeans.java @@ -105,6 +105,8 @@ final class BackendBeans { static final String[] LISTENERS = {MESSAGE_LISTENER, KAFKA_LISTENER, RABBIT_LISTENER, JMS_LISTENER}; static final String MESSAGE_TEMPLATE_TYPE = "com/codename1/backend/messaging/MessageTemplate"; + /// Injected to send mail; see [#usesMail]. + static final String MAIL_SENDER_TYPE = "com/codename1/backend/mail/MailSender"; static final String MESSAGE_TYPE = "com/codename1/backend/messaging/Message"; static final String CACHEABLE = PKG + "Cacheable;"; static final String CACHE_PUT = PKG + "CachePut;"; @@ -517,6 +519,7 @@ static final class Aspect { boolean needsDatabase; /// Whether a bean injects the MessageTemplate. boolean usesMessageTemplate; + boolean usesMailSender; boolean needsEntities; boolean needsSession; /// The `@Repository` interfaces injected anywhere, by internal name, each @@ -810,6 +813,24 @@ boolean usesMessaging() { return usesMessageTemplate || hasListeners(); } + /// Whether the module sends mail: a bean is given a `MailSender`, or the + /// configuration names a mail server -- which code that asks for + /// `MailSender.current()`, and the security flows that send, rely on with + /// no bean taking one. + boolean usesMail() { + return usesMailSender + || RestControllerAnnotationProcessor.applicationPropertyKnown(ctx, "cn1.mail.url") + || RestControllerAnnotationProcessor.applicationPropertyKnown(ctx, + "cn1.mail.transports") + || RestControllerAnnotationProcessor.applicationPropertyKnown(ctx, + "spring.mail.host"); + } + + /// [#mailTransports] of this module. + List mailTransports() { + return mailTransports(ctx); + } + /// [#messagingTransports] of this module. List transports() { return messagingTransports(ctx); @@ -1938,6 +1959,66 @@ static List messagingTransports(ProcessorContext ctx) { return out; } + /// The mail transports the module's configuration names, as the factory + /// classes the wiring registers, by the rules of [#messagingTransports]: + /// the scheme of `cn1.mail.url`, SMTP for Spring Boot's `spring.mail.host`, + /// every one for a URL the build cannot read or a configuration that names + /// none, and exactly the set `cn1.mail.transports` lists when it is there. + static List mailTransports(ProcessorContext ctx) { + Set schemes = new LinkedHashSet(); + for (String list : RestControllerAnnotationProcessor.applicationPropertyValues(ctx, + "cn1.mail.transports")) { + int from = 0; + while (from <= list.length()) { + int comma = list.indexOf(',', from); + int end = comma < 0 ? list.length() : comma; + String name = list.substring(from, end).trim(); + if (name.length() > 0) { + schemes.add(name); + } + from = end + 1; + } + } + boolean named = !schemes.isEmpty(); + for (String url : RestControllerAnnotationProcessor.applicationPropertyValues(ctx, + "cn1.mail.url")) { + if (url.indexOf("${") >= 0) { + schemes.addAll(MAIL_TRANSPORTS.keySet()); + continue; + } + int colon = url.indexOf(':'); + String scheme = colon < 0 ? url : url.substring(0, colon); + if (!MAIL_TRANSPORTS.containsKey(scheme) && scheme.endsWith("s")) { + scheme = scheme.substring(0, scheme.length() - 1); + } + schemes.add(scheme); + } + if (RestControllerAnnotationProcessor.applicationPropertyKnown(ctx, "spring.mail.host")) { + schemes.add("smtp"); + } + if (!named && schemes.isEmpty() && !RestControllerAnnotationProcessor + .applicationPropertyKnown(ctx, "cn1.mail.url")) { + schemes.addAll(MAIL_TRANSPORTS.keySet()); + } + List out = new ArrayList(); + for (String scheme : schemes) { + String factory = MAIL_TRANSPORTS.get(scheme); + if (factory != null) { + out.add(scheme + " " + factory); + } + } + return out; + } + + /// The mail transports by scheme. `memory:` and `log:` are part of the + /// mail package itself. + static final Map MAIL_TRANSPORTS = new LinkedHashMap(); + + static { + MAIL_TRANSPORTS.put("smtp", "com.codename1.backend.mail.smtp.SmtpTransport.Factory"); + MAIL_TRANSPORTS.put("ses", "com.codename1.backend.mail.ses.SesTransport.Factory"); + } + /// The broker transports by scheme. `memory:` and `db:` are part of the /// messaging package itself. static final Map TRANSPORTS = new LinkedHashMap(); @@ -2635,6 +2716,11 @@ private void resolvePoint(Bean owner, Point p, AnnotatedClass where) { usesMessageTemplate = true; return; } + if (MAIL_SENDER_TYPE.equals(type)) { + p.builtin = "mailSender"; + usesMailSender = true; + return; + } if (DATASOURCE_TYPE.equals(type)) { p.builtin = "dataSource"; needsDatabase = true; diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendWiringWriter.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendWiringWriter.java index 88bfccb5972..ea1815416d9 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendWiringWriter.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/BackendWiringWriter.java @@ -234,6 +234,30 @@ private void create(StringBuilder sb, List routers) { .append(t.substring(space + 1)).append("());\n"); } } + if (MvcTemplates.hasMailTemplates(model.ctx)) { + // The compiled templates are a class of the application's, which + // the mail support cannot name; this is how it reaches them. + sb.append(" com.codename1.impl.backend.MailRuntime.install(\n" + + " new com.codename1.impl.backend.MailRuntime.TemplateRenderer() {\n" + + " @Override\n" + + " public byte[] render(String view, " + + "com.codename1.backend.mvc.Model model) {\n" + + " return com.codename1.generated.mvc.Views.render(view, model);\n" + + " }\n" + + " });\n"); + } + if (model.usesMail()) { + // As for messaging: the outbox is kept in the server's data + // source, and only the transports the configuration names are + // referenced, so only those are compiled in. + sb.append(" com.codename1.backend.mail.Mailers.bindDataSource(dataSource);\n"); + for (String t : model.mailTransports()) { + int space = t.indexOf(' '); + sb.append(" com.codename1.backend.mail.Mailers.register(") + .append(BackendSources.quote(t.substring(0, space))).append(", new ") + .append(t.substring(space + 1)).append("());\n"); + } + } sb.append(" entities = environment.getEntityManager();\n"); if (model.needsSession) { sb.append(" transactionSession = new com.codename1.impl.backend." @@ -607,6 +631,9 @@ String point(BackendBeans.Point p) { if ("session".equals(p.builtin)) { return "transactionSession"; } + if ("mailSender".equals(p.builtin)) { + return "new com.codename1.backend.mail.MailSender()"; + } if ("messageTemplate".equals(p.builtin)) { return "new com.codename1.backend.messaging.MessageTemplate()"; } @@ -860,6 +887,15 @@ private void started(StringBuilder sb) { if (model.hasListeners()) { listeners(sb); } + if (model.usesMail()) { + // Opened now and not at the first send: a mail the last run left + // in the outbox is sent by this one whether or not it sends any + // of its own. + sb.append(" if (dataSource != null " + + "&& com.codename1.backend.mail.Mailers.isConfigured(config)) {\n" + + " com.codename1.backend.mail.Mailers.current();\n" + + " }\n"); + } // Then the runners, by @Order, and the ready listeners after them, // as Spring Boot runs them once the context is up. List ordered = new ArrayList(model.beans); diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/MvcTemplates.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/MvcTemplates.java index c509519e6aa..b3c8f4fe2b0 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/MvcTemplates.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/MvcTemplates.java @@ -108,6 +108,26 @@ static File projectDirectory(ProcessorContext ctx) { return ctx.getProjectDir(); } + /// Whether the module has templates for mail -- an `.html` anywhere under + /// `src/main/resources/templates/mail` -- so the views are compiled for a + /// module no controller of which returns one, and the wiring gives them + /// to the mail support. + static boolean hasMailTemplates(ProcessorContext ctx) { + File project = projectDirectory(ctx); + return project != null + && hasHtml(new File(project, "src/main/resources/templates/mail"), 0); + } + + private static boolean hasHtml(File directory, int depth) { + File[] files = directory.listFiles(); + if (files == null || depth > 16) return false; + for (File file : files) { + if (file.isDirectory() ? hasHtml(file, depth + 1) : file.getName().endsWith(".html")) + return true; + } + return false; + } + Map sources() { File project = projectDirectory(ctx); if (project == null) diff --git a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java index 3265f74d830..864b572933f 100644 --- a/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java +++ b/maven/build-engine/src/main/java/com/codename1/maven/processors/RestControllerAnnotationProcessor.java @@ -1877,15 +1877,19 @@ public void finish(ProcessorContext ctx) throws ProcessingException { sources.putAll(extra); boolean hasViews = false; for (Controller controller : controllers.values()) for (Route route : controller.routes) hasViews |= route.view; - if (hasViews || hasMvcControllers()) { + // A module whose only templates are its mails has no route that + // returns a view; they are compiled all the same, and nothing of the + // MVC support for requests (the assets) comes with them. + boolean mailViews = MvcTemplates.hasMailTemplates(ctx); + if (hasViews || hasMvcControllers() || mailViews) { try { - Map mvcSources = hasViews ? new MvcTemplates(ctx).sources() : new LinkedHashMap(); - mvcSources.putAll(MvcAssets.sources(MvcTemplates.projectDirectory(ctx))); + Map mvcSources = hasViews || mailViews ? new MvcTemplates(ctx).sources() : new LinkedHashMap(); + if (hasViews || hasMvcControllers()) mvcSources.putAll(MvcAssets.sources(MvcTemplates.projectDirectory(ctx))); for (String generated : mvcSources.keySet()) if (isNotOurOwnOutput(ctx, generated)) { ctx.error("Generated MVC class would overwrite " + generated); return; } sources.putAll(mvcSources); - ctx.setAttribute("cn1.backend.mvc", Boolean.TRUE); + if (hasViews || hasMvcControllers()) ctx.setAttribute("cn1.backend.mvc", Boolean.TRUE); } catch (IllegalArgumentException error) { ctx.error(error.getMessage()); return; } } diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/BackendBeansTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/BackendBeansTest.java index 38a6ff8184f..05964e3e752 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/BackendBeansTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/BackendBeansTest.java @@ -1635,6 +1635,52 @@ public void messagingRegistersOnlyTheTransportsTheConfigurationNames() throws Ex } } + /// Mail is wired by the same rule: the transport the configuration names + /// and no other, for a bean that is given a `MailSender` or a + /// configuration that names a mail server -- and nothing of it otherwise. + @Test + public void mailRegistersOnlyTheTransportsTheConfigurationNames() throws Exception { + String sender = PKG + "@Component public class Welcome {\n" + + " private final com.codename1.backend.mail.MailSender mail;\n" + + " public Welcome(com.codename1.backend.mail.MailSender mail) { this.mail = mail; }\n" + + "}\n"; + // {injects a MailSender, properties, transports registered, wired at all} + String[][] cases = { + {"y", "", "smtp ses", "y"}, + {"y", "cn1.mail.url=memory:\n", "", "y"}, + {"y", "cn1.mail.url=smtp://mail.example.com:587\n", "smtp", "y"}, + {"y", "cn1.mail.url=smtps://mail.example.com\n", "smtp", "y"}, + {"y", "cn1.mail.url=ses://eu-west-1\n", "ses", "y"}, + {"y", "spring.mail.host=mail.example.com\n", "smtp", "y"}, + {"y", "cn1.mail.url=${MAIL_URL}\n", "smtp ses", "y"}, + {"y", "cn1.mail.transports=ses\n", "ses", "y"}, + // No bean takes one, and a server is named: MailSender.current() + // and the security flows still have to find it. + {"n", "cn1.mail.url=smtp://mail.example.com\n", "smtp", "y"}, + {"n", "", "", "n"}, + }; + for (String[] c : cases) { + Map s = new LinkedHashMap(); + s.put("com.example.Inbox", PKG + "@Component public class Inbox {\n" + + " @MessageListener(\"q\") public void on(String text) { }\n" + + "}\n"); + if ("y".equals(c[0])) { + s.put("com.example.Welcome", sender); + } + File classes = compile(s); + writeProperties(classes, "application.properties", c[1]); + assertNoErrors(process(classes)); + String wiring = new String(Files.readAllBytes(new File(classes, + "com/example/BackendWiring.class").toPath()), "ISO-8859-1"); + String what = c[0] + " " + c[1]; + assertEquals(what, c[2].indexOf("smtp") >= 0, + wiring.indexOf("mail/smtp/SmtpTransport") >= 0); + assertEquals(what, c[2].indexOf("ses") >= 0, + wiring.indexOf("mail/ses/SesTransport") >= 0); + assertEquals(what, "y".equals(c[3]), wiring.indexOf("backend/mail/Mailers") >= 0); + } + } + @Test public void messageListenersReceiveRetryAndDeadLetter() throws Exception { Map s = new LinkedHashMap(); diff --git a/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java b/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java index cc0e2dd2c35..95a71d515a7 100644 --- a/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java +++ b/maven/build-engine/src/test/java/com/codename1/maven/processors/MigrationGeneratorTest.java @@ -338,7 +338,7 @@ public void theGoalsApplyTheSecurityTablesAModuleAskedFor() throws Exception { try { assertEquals(Long.valueOf(0), pool.queryOne("SELECT COUNT(*) AS n FROM cn1_users", null) .get("n")); - assertEquals(Long.valueOf(12), pool.queryOne( + assertEquals(Long.valueOf(13), pool.queryOne( "SELECT COUNT(*) AS n FROM cn1_security_schema_history", null).get("n")); assertEquals(Long.valueOf(1), pool.queryOne("SELECT COUNT(*) AS n FROM profiles", null) .get("n")); @@ -393,24 +393,26 @@ public void aModuleWhoseOnlyMigrationsAreTheSecurityTablesGetsTheGoals() throws File database = new File(tmp.newFolder(), "cli-library-only.db"); String migrated = goal(database, "migrate"); - // Each of the eleven under a name of its own. + // Each version under a name of its own. java.util.Set names = new java.util.TreeSet(); for (String line : migrated.split("\n")) { if (line.startsWith("cn1: security: applied ")) { names.add(line.substring("cn1: security: applied ".length()).trim()); } } - assertEquals(migrated, 12, names.size()); + assertEquals(migrated, 13, names.size()); assertTrue(migrated, names.contains( "com.codename1.backend.security.SecuritySchema.V5__rate_limits")); assertTrue(migrated, names.contains( "com.codename1.backend.security.SecuritySchema.V11__remember_me_rotation_grace")); assertTrue(migrated, names.contains( "com.codename1.backend.security.SecuritySchema.V12__remember_me_devices")); - assertTrue(migrated, migrated.contains("cn1: security: 12 applied, schema at version 12")); + assertTrue(migrated, names.contains( + "com.codename1.backend.security.SecuritySchema.V13__mail_tokens")); + assertTrue(migrated, migrated.contains("cn1: security: 13 applied, schema at version 13")); String info = goal(database, "info"); - assertTrue(info, info.contains("cn1: security: 12 migration(s) in cn1_security_schema_history")); + assertTrue(info, info.contains("cn1: security: 13 migration(s) in cn1_security_schema_history")); assertTrue(goal(database, "validate"), goal(database, "validate").contains( "cn1: security: the schema history matches this build's migrations")); assertTrue(goal(database, "migrate").contains("cn1: security: nothing to apply")); diff --git a/vm/backend/demo/mailcheck/com/demo/MailCheck.java b/vm/backend/demo/mailcheck/com/demo/MailCheck.java new file mode 100644 index 00000000000..0ed5610ec67 --- /dev/null +++ b/vm/backend/demo/mailcheck/com/demo/MailCheck.java @@ -0,0 +1,1288 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import com.codename1.backend.Base64; +import com.codename1.backend.Config; +import com.codename1.backend.DataSource; +import com.codename1.backend.ServerSocket; +import com.codename1.backend.Transactions; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailAuthenticationException; +import com.codename1.backend.mail.MailException; +import com.codename1.backend.mail.MailSendException; +import com.codename1.backend.mail.Mailer; +import com.codename1.backend.mail.Mailers; +import com.codename1.backend.mail.smtp.SmtpTransport; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/// Exercises mail: the MIME encoder, the SMTP client against a mail server +/// that runs inside this process and records the bytes it is sent, the queue, +/// its retries and the outbox. The same checks run on the JVM and on the +/// native build. `CN1_MAILCHECK_URL` (or the first argument) names a real +/// server to send one mail of every shape through as well; its recipients +/// are `CN1_MAILCHECK_TO`. +public class MailCheck { + private static int passed; + private static final List failures = new ArrayList(); + /// e acute, built so that this file stays ASCII. + private static final String ACUTE = String.valueOf((char) 0xe9); + + public static void main(String[] args) throws Exception { + Mailers.register("smtp", new SmtpTransport.Factory()); + validation(); + capture(); + Server server = new Server(); + try { + encoding(server); + session(server); + refusals(server); + security(server); + queue(server, null); + String file = "mailcheck-" + Long.toString(System.nanoTime(), 36) + ".db"; + queue(server, file); + outbox(server, file); + passed += MailFlows.run(null, failures); + passed += SesCheck.run(failures); + passed += MailFlows.run(file, failures); + new java.io.File(file).delete(); + new java.io.File(file + "-shm").delete(); + new java.io.File(file + "-wal").delete(); + String db = System.getenv("CN1_MAILCHECK_DB"); + if (db != null && db.length() > 0) { + passed += MailFlows.run(db, failures); + queue(server, db); + outbox(server, db); + } + } finally { + server.stop(); + } + String url = args.length > 0 ? args[0] : System.getenv("CN1_MAILCHECK_URL"); + if (url != null && url.length() > 0) { + real(url); + } + System.out.println("passed=" + passed + " failed=" + failures.size()); + for (String f : failures) { + System.out.println("FAIL " + f); + } + System.out.println(failures.isEmpty() ? "MAILCHECK OK" : "MAILCHECK FAILED"); + System.exit(failures.isEmpty() ? 0 : 1); + } + + // ---------------------------------------------------------------- checks + + private static void validation() { + check("no recipient", Boolean.TRUE, Boolean.valueOf(rejects(Mail.builder().subject("x") + .text("y")))); + check("no body", Boolean.TRUE, Boolean.valueOf(rejects(Mail.to("a@example.com") + .subject("x")))); + check("bad address", Boolean.TRUE, Boolean.valueOf(rejectsCall(new Runnable() { + public void run() { + Mail.to("not an address").subject("s").text("t").build(); + } + }))); + check("header injection in subject", Boolean.TRUE, Boolean.valueOf(rejectsCall( + new Runnable() { + public void run() { + Mail.to("a@example.com").subject("hi\r\nBcc: evil@example.com").text("t").build(); + } + }))); + check("header injection in address", Boolean.TRUE, Boolean.valueOf(rejectsCall( + new Runnable() { + public void run() { + Mail.to("a@example.com\r\nRCPT TO:").subject("s").text("t").build(); + } + }))); + check("header injection in header", Boolean.TRUE, Boolean.valueOf(rejectsCall( + new Runnable() { + public void run() { + Mail.to("a@example.com").header("X-Tag", "a\nb").subject("s").text("t").build(); + } + }))); + check("reserved header", Boolean.TRUE, Boolean.valueOf(rejectsCall(new Runnable() { + public void run() { + Mail.to("a@example.com").header("Subject", "two").subject("s").text("t").build(); + } + }))); + check("a header no line could hold", Boolean.TRUE, Boolean.valueOf(rejectsCall( + new Runnable() { + public void run() { + StringBuilder url = new StringBuilder("") + .subject("s").text("t").build(); + } + }))); + check("angle address in recipient", Boolean.TRUE, Boolean.valueOf(rejectsCall( + new Runnable() { + public void run() { + Mail.to("a@example.com> SIZE=1 "); + Mailer m = Mailers.open(Config.of(p, "test"), null); + try { + m.send(Mail.to("a@example.com").cc("c@example.com").subject("Hello") + .html("

Hi there

Open

") + .attach("a.txt", "text/plain", WireBuffer.utf8("file")).build()); + List sent = m.sent(); + check("captured count", Integer.valueOf(1), Integer.valueOf(sent.size())); + Mail got = sent.get(0); + check("captured from", "Shop ", got.getFrom()); + check("captured to", "[a@example.com]", String.valueOf(got.getTo())); + check("captured cc", "[c@example.com]", String.valueOf(got.getCc())); + check("captured subject", "Hello", got.getSubject()); + check("captured derived text", "Hi there\n\nOpen (https://example.com/x)\n", + got.getText()); + check("captured attachment names", "[a.txt]", + String.valueOf(got.getAttachmentNames())); + check("captured attachment", "file", WireBuffer.fromUtf8(got.getAttachment("a.txt"))); + m.clearSent(); + check("cleared", Integer.valueOf(0), Integer.valueOf(m.sent().size())); + // A < that opens no tag is a sign, and what follows it is text. + m.send(Mail.to("a@example.com").subject("s") + .html("

ok

Total < 5 items, 3 > 2 and a <").build()); + String derived = m.sent().get(0).getText(); + check("a less-than sign is text", Boolean.TRUE, Boolean.valueOf(derived != null + && derived.indexOf("Total < 5 items, 3 > 2 and a <") >= 0)); + m.clearSent(); + boolean threw = false; + try { + Mailers.open(Config.of(new Properties(), "test"), null) + .sendNow(Mail.to("a@example.com").subject("s").text("t").build()); + } catch (IOException err) { + threw = true; + } + check("no sender is an error", Boolean.TRUE, Boolean.valueOf(threw)); + } finally { + m.close(); + } + } + + private static Mailer mailer(Server server, String database, DataSource[] ds, + String... settings) throws Exception { + Properties p = new Properties(); + p.setProperty("cn1.mail.url", "smtp://127.0.0.1:" + server.port); + p.setProperty("cn1.mail.smtp.starttls", "false"); + p.setProperty("cn1.mail.from", "Shop "); + p.setProperty("cn1.mail.domain", "example.com"); + p.setProperty("cn1.mail.retry.backoff", "30"); + p.setProperty("cn1.mail.retry.multiplier", "1"); + p.setProperty("cn1.mail.retry.max-attempts", "3"); + p.setProperty("cn1.mail.outbox.sweep", "200"); + p.setProperty("cn1.mail.timeout.read", "5000"); + for (int i = 0; i < settings.length; i += 2) { + p.setProperty(settings[i], settings[i + 1]); + } + DataSource d = database == null ? null : DataSource.open(database, 4); + if (ds != null) { + ds[0] = d; + } + return Mailers.open(Config.of(p, "test"), d); + } + + /// What the encoder writes, read back the way a mail reader would. + private static void encoding(Server server) throws Exception { + server.reset(); + Mailer m = mailer(server, null, null); + try { + // Plain ASCII: nothing encoded, nothing folded. + m.sendNow(Mail.to("a@example.com").subject("Plain").text("line one\nline two\n") + .build()); + Received r = server.last(); + check("plain from", "shop@example.com", r.from); + check("plain rcpt", "[a@example.com]", String.valueOf(r.recipients)); + check("plain From", "Shop ", r.header("From")); + check("plain To", "a@example.com", r.header("To")); + check("plain Subject", "Plain", r.header("Subject")); + check("plain MIME-Version", "1.0", r.header("MIME-Version")); + check("plain type", "text/plain; charset=UTF-8", r.header("Content-Type")); + check("plain encoding", "7bit", r.header("Content-Transfer-Encoding")); + check("plain body", "line one\r\nline two\r\n", r.body()); + String id = r.header("Message-ID"); + check("message id", Boolean.TRUE, Boolean.valueOf(id != null && id.startsWith("<") + && id.endsWith("@example.com>") && id.length() == 46)); + String date = r.header("Date"); + check("date", Boolean.TRUE, Boolean.valueOf(date != null && date.endsWith(" +0000") + && date.charAt(3) == ',')); + check("plain wire form", "", r.wireProblems()); + + // A subject and a name with no place to fold are written so + // that they have one: no line passes what a server takes. + StringBuilder word = new StringBuilder(); + while (word.length() < 1500) { + word.append("unbroken"); + } + StringBuilder longName = new StringBuilder(); + while (longName.length() < 1100) { + longName.append("Long Name "); + } + m.sendNow(Mail.to(longName.toString().trim() + " ") + .subject(word.toString()).text("t").build()); + r = server.last(); + check("long word wire form", "", r.wireProblems()); + check("long word subject", word.toString(), decodeWords(r.header("Subject"))); + check("long name", longName.toString().trim() + " ", + decodeWords(r.header("To"))); + + // Dots at the start of a line survive the trip. + String dotted = ".\n..two\n.three.\nfour\n."; + m.sendNow(Mail.to("a@example.com").subject("Dots").text(dotted).build()); + check("dot stuffing", ".\r\n..two\r\n.three.\r\nfour\r\n.\r\n", server.last().body()); + + // Non-ASCII everywhere it may appear. + String name = "Ren" + ACUTE + "e"; + StringBuilder longSubject = new StringBuilder(); + for (int i = 0; i < 30; i++) { + longSubject.append("caf").append(ACUTE).append(' '); + } + String subject = longSubject.toString().trim(); + StringBuilder longLine = new StringBuilder(); + for (int i = 0; i < 40; i++) { + longLine.append("d").append(ACUTE).append("j").append((char) 0xe0).append(" vu = "); + } + String text = longLine + "\ntrailing space \nFrom the start\n"; + m.sendNow(Mail.to(name + " ", "b@example.com").subject(subject) + .text(text).build()); + r = server.last(); + check("utf8 wire form", "", r.wireProblems()); + check("utf8 To", name + " , b@example.com", + decodeWords(r.header("To"))); + check("utf8 Subject", subject, decodeWords(r.header("Subject"))); + check("utf8 encoding", "quoted-printable", r.header("Content-Transfer-Encoding")); + check("utf8 body", text.replace("\n", "\r\n"), + WireBuffer.fromUtf8(quotedPrintable(r.body()))); + + // A long ASCII header is folded, never encoded. + StringBuilder ascii = new StringBuilder(); + for (int i = 0; i < 40; i++) { + ascii.append("word").append(i).append(' '); + } + m.sendNow(Mail.to("a@example.com").subject(ascii.toString().trim()).text("x").build()); + r = server.last(); + check("folded wire form", "", r.wireProblems()); + check("folded Subject", ascii.toString().trim(), r.header("Subject")); + check("folded, not encoded", Boolean.FALSE, + Boolean.valueOf(r.rawHeader("Subject").indexOf("=?") >= 0)); + + // Text and HTML, an inline image and two attachments. + byte[] image = new byte[5000]; + for (int i = 0; i < image.length; i++) { + image[i] = (byte) (i * 31 + 7); + } + byte[] empty = new byte[0]; + String fileName = "r" + ACUTE + "sum" + ACUTE + " final.pdf"; + m.sendNow(Mail.to("a@example.com").bcc("hidden@example.com").replyTo("help@example.com") + .subject("Parts").text("see the picture") + .html("

see

") + .inline("logo", "image/png", image) + .attach(fileName, "application/pdf", image) + .attach("empty.bin", "application/octet-stream", empty) + .header("X-Campaign", "spring").build()); + r = server.last(); + check("parts wire form", "", r.wireProblems()); + check("bcc is a recipient", "[a@example.com, hidden@example.com]", + String.valueOf(r.recipients)); + check("bcc is not a header", Boolean.FALSE, + Boolean.valueOf(r.data.indexOf("hidden@example.com") >= 0)); + check("reply to", "help@example.com", r.header("Reply-To")); + check("custom header", "spring", r.header("X-Campaign")); + Part root = Part.parse(r.data); + check("mixed", Boolean.TRUE, Boolean.valueOf(root.type.startsWith("multipart/mixed"))); + check("mixed parts", Integer.valueOf(3), Integer.valueOf(root.parts.size())); + Part alt = root.parts.size() > 0 ? root.parts.get(0) : new Part(); + check("alternative", Boolean.TRUE, + Boolean.valueOf(alt.type.startsWith("multipart/alternative"))); + check("alternative parts", Integer.valueOf(2), Integer.valueOf(alt.parts.size())); + Part related = alt.parts.size() == 2 ? alt.parts.get(1) : new Part(); + check("related", Boolean.TRUE, + Boolean.valueOf(related.type.startsWith("multipart/related"))); + check("related parts", Integer.valueOf(2), Integer.valueOf(related.parts.size())); + if (alt.parts.size() == 2) { + check("text first", "text/plain; charset=UTF-8", alt.parts.get(0).type); + check("text part", "see the picture", alt.parts.get(0).text().trim()); + } + if (related.parts.size() == 2) { + check("html in related", "text/html; charset=UTF-8", related.parts.get(0).type); + check("html part", "

see

", + related.parts.get(0).text().trim()); + Part img = related.parts.get(1); + check("inline id", "", img.header("Content-ID")); + check("inline disposition", Boolean.TRUE, Boolean.valueOf( + String.valueOf(img.header("Content-Disposition")).startsWith("inline"))); + check("inline bytes", Boolean.TRUE, + Boolean.valueOf(java.util.Arrays.equals(image, img.bytes()))); + } + if (root.parts.size() == 3) { + Part pdf = root.parts.get(1); + check("attachment type", Boolean.TRUE, + Boolean.valueOf(pdf.type.startsWith("application/pdf"))); + check("attachment bytes", Boolean.TRUE, + Boolean.valueOf(java.util.Arrays.equals(image, pdf.bytes()))); + String disposition = String.valueOf(pdf.header("Content-Disposition")); + check("attachment name", Boolean.TRUE, Boolean.valueOf(disposition.indexOf( + "filename*=UTF-8''r%C3%A9sum%C3%A9%20final.pdf") >= 0)); + check("empty attachment", Integer.valueOf(0), + Integer.valueOf(root.parts.get(2).bytes().length)); + } + } finally { + m.close(); + } + } + + /// How the client speaks: one write for the envelope, one connection for + /// several mails, and the same result without pipelining. + private static void session(Server server) throws Exception { + server.reset(); + Mailer m = mailer(server, null, null); + try { + for (int i = 0; i < 3; i++) { + m.sendNow(Mail.to("a@example.com", "b@example.com").subject("n" + i).text("t") + .build()); + } + check("one connection for three mails", Integer.valueOf(1), + Integer.valueOf(server.connections.get())); + check("three mails", Integer.valueOf(3), Integer.valueOf(server.count())); + check("envelope in one write", Boolean.TRUE, Boolean.valueOf(server.pipelined.get())); + check("size declared", Boolean.TRUE, Boolean.valueOf(server.sawSize.get())); + } finally { + m.close(); + } + server.reset(); + server.pipelining = false; + m = mailer(server, null, null); + try { + m.sendNow(Mail.to("a@example.com", "b@example.com").subject("one by one").text("t") + .build()); + check("without pipelining", "[a@example.com, b@example.com]", + String.valueOf(server.last().recipients)); + check("not pipelined", Boolean.FALSE, Boolean.valueOf(server.pipelined.get())); + // A big body leaves in pieces and arrives whole. + StringBuilder big = new StringBuilder(); + for (int i = 0; i < 20000; i++) { + big.append(i % 7 == 0 ? "." : "").append("line ").append(i).append('\n'); + } + m.sendNow(Mail.to("a@example.com").subject("big").text(big.toString()).build()); + check("big body", big.toString().replace("\n", "\r\n"), server.last().body()); + } finally { + m.close(); + } + } + + private static MailException failure(Mailer m, Mail mail) throws IOException { + try { + m.sendNow(mail); + return null; + } catch (MailException err) { + return err; + } + } + + private static void refusals(Server server) throws Exception { + for (int round = 0; round < 2; round++) { + server.reset(); + server.pipelining = round == 0; + String w = round == 0 ? "pipelined " : "sequential "; + Mailer m = mailer(server, null, null); + try { + MailException err = failure(m, Mail.to("reject@example.com").subject("s").text("t") + .build()); + check(w + "refused recipient is permanent", Boolean.TRUE, + Boolean.valueOf(err != null && err.isPermanent())); + check(w + "refused code", Integer.valueOf(550), err instanceof MailSendException + ? Integer.valueOf(((MailSendException) err).getCode()) : null); + err = failure(m, Mail.to("a@example.com", "defer@example.com").subject("s").text("t") + .build()); + check(w + "deferred recipient is transient", Boolean.FALSE, + err == null ? null : Boolean.valueOf(err.isPermanent())); + check(w + "nothing sent when one is deferred", Integer.valueOf(0), + Integer.valueOf(server.count())); + err = failure(m, Mail.to("a@example.com", "reject@example.com").subject("s") + .text("t").build()); + check(w + "partly refused is permanent", Boolean.TRUE, + Boolean.valueOf(err != null && err.isPermanent())); + check(w + "partly refused is sent to the rest", "[a@example.com]", + server.count() == 1 ? String.valueOf(server.last().recipients) : null); + // The session is still usable after each of those. + m.sendNow(Mail.to("a@example.com").subject("after").text("t").build()); + check(w + "session survives refusals", Integer.valueOf(2), + Integer.valueOf(server.count())); + server.failData.set(1); + err = failure(m, Mail.to("a@example.com").subject("s").text("t").build()); + check(w + "451 after the body is transient", Boolean.FALSE, + err == null ? null : Boolean.valueOf(err.isPermanent())); + server.rejectData.set(1); + err = failure(m, Mail.to("a@example.com").subject("s").text("t").build()); + check(w + "554 after the body is permanent", Boolean.TRUE, + Boolean.valueOf(err != null && err.isPermanent())); + server.maxSize = 2000; + } finally { + m.close(); + } + m = mailer(server, null, null); + try { + int before = server.count(); + MailException err = failure(m, Mail.to("a@example.com").subject("s") + .attach("big.bin", "application/octet-stream", new byte[4000]).text("t") + .build()); + check(w + "too large is permanent", Boolean.TRUE, + Boolean.valueOf(err != null && err.isPermanent())); + check(w + "too large sends nothing", Integer.valueOf(before), + Integer.valueOf(server.count())); + String address = "jos" + ACUTE + "@example.com"; + err = failure(m, Mail.to(address).subject("s").text("t").build()); + check(w + "utf8 address without SMTPUTF8", Boolean.TRUE, + Boolean.valueOf(err != null && err.isPermanent())); + server.maxSize = 0; + } finally { + m.close(); + } + server.smtpUtf8 = true; + m = mailer(server, null, null); + try { + String address = "jos" + ACUTE + "@example.com"; + m.sendNow(Mail.to(address).subject("s").text("t").build()); + check(w + "utf8 address", "[" + address + "]", + String.valueOf(server.last().recipients)); + check(w + "SMTPUTF8 asked for", Boolean.TRUE, + Boolean.valueOf(server.sawUtf8.get())); + } finally { + m.close(); + } + // A sender whose domain is not ASCII, and no cn1.mail.domain: the + // Message-ID is still one a header may hold. + m = mailer(server, null, null, "cn1.mail.domain", ""); + try { + m.sendNow(Mail.to("a@example.com").from("shop@caf" + ACUTE + ".example") + .subject("s").text("t").build()); + String id = server.last().header("Message-ID"); + check(w + "message id of a sender outside ASCII", Boolean.TRUE, + Boolean.valueOf(id != null && id.endsWith("@mail.invalid>"))); + check(w + "and the headers after it are headers", "1.0", + server.last().header("MIME-Version")); + } finally { + m.close(); + } + } + // Nothing listening. + Properties p = new Properties(); + p.setProperty("cn1.mail.url", "smtp://127.0.0.1:1"); + p.setProperty("cn1.mail.from", "shop@example.com"); + Mailer m = Mailers.open(Config.of(p, "test"), null); + try { + MailException err = failure(m, Mail.to("a@example.com").subject("s").text("t") + .build()); + check("unreachable is transient", Boolean.FALSE, + err == null ? null : Boolean.valueOf(err.isPermanent())); + check("unreachable is unhealthy", Boolean.FALSE, Boolean.valueOf(m.isHealthy())); + } finally { + m.close(); + } + } + + private static void security(Server server) throws Exception { + Mail mail = Mail.to("a@example.com").subject("s").text("t").build(); + // STARTTLS is required unless switched off. + server.reset(); + Mailer m = mailer(server, null, null, "cn1.mail.smtp.starttls", "true"); + try { + MailException err = failure(m, mail); + check("no STARTTLS is refused", Boolean.TRUE, Boolean.valueOf(err != null + && String.valueOf(err.getMessage()).indexOf("STARTTLS") >= 0)); + check("no STARTTLS sends nothing", Integer.valueOf(0), + Integer.valueOf(server.count())); + } finally { + m.close(); + } + // A response injected between the go-ahead and the handshake. + server.reset(); + server.startTls = true; + server.inject = true; + m = mailer(server, null, null, "cn1.mail.smtp.starttls", "true"); + try { + MailException err = failure(m, mail); + check("injection after STARTTLS is refused", Boolean.TRUE, Boolean.valueOf(err != null + && String.valueOf(err.getMessage()).indexOf("inject") >= 0)); + check("injection sends nothing", Integer.valueOf(0), Integer.valueOf(server.count())); + } finally { + m.close(); + } + // A password never crosses a connection that is not encrypted. + server.reset(); + Properties p = new Properties(); + p.setProperty("cn1.mail.url", "smtp://user:p%40ss@127.0.0.1:" + server.port); + p.setProperty("cn1.mail.smtp.starttls", "false"); + p.setProperty("cn1.mail.from", "shop@example.com"); + m = Mailers.open(Config.of(p, "test"), null); + try { + MailException err = failure(m, mail); + check("password in the clear is refused", Boolean.TRUE, + Boolean.valueOf(err instanceof MailAuthenticationException)); + check("no AUTH was sent", Boolean.FALSE, Boolean.valueOf(server.sawAuth.get())); + } finally { + m.close(); + } + } + + private static boolean await(Server server, int count) throws InterruptedException { + long end = System.currentTimeMillis() + 10000; + while (server.count() < count && System.currentTimeMillis() < end) { + Thread.sleep(10); + } + return server.count() >= count; + } + + private static boolean idle(Mailer m) throws InterruptedException { + long end = System.currentTimeMillis() + 10000; + while (m.pending() > 0 && System.currentTimeMillis() < end) { + Thread.sleep(10); + } + return m.pending() == 0; + } + + private static long rows(DataSource ds, String table) throws IOException { + List rows = ds.query("SELECT COUNT(*) AS n FROM " + table, new Object[0]); + return ((Number) ((Map) rows.get(0)).get("n")).longValue(); + } + + /// The queue: send returns at once, a deferred mail is tried again, a + /// refused one is not, and one that keeps failing is given up on. + private static void queue(Server server, String database) throws Exception { + String w = database == null ? "memory queue " : database.indexOf("://") > 0 + ? database.substring(0, database.indexOf(':')) + " queue " : "sqlite queue "; + server.reset(); + DataSource[] ds = new DataSource[1]; + Mailer m = mailer(server, database, ds); + try { + if (ds[0] != null) { + clear(ds[0]); + } + for (int i = 0; i < 5; i++) { + m.send(Mail.to("a@example.com").subject("q" + i).text("t").build()); + } + check(w + "delivers", Boolean.TRUE, Boolean.valueOf(await(server, 5))); + check(w + "in order", "q0 q1 q2 q3 q4", server.subjects()); + check(w + "drains", Boolean.TRUE, Boolean.valueOf(idle(m))); + + server.reset(); + server.failData.set(2); + m.send(Mail.to("a@example.com").subject("retried").text("t").build()); + check(w + "retries a deferred mail", Boolean.TRUE, Boolean.valueOf(await(server, 1))); + check(w + "third attempt", Integer.valueOf(3), Integer.valueOf(server.bodies.get())); + + server.reset(); + server.rejectData.set(1); + m.send(Mail.to("a@example.com").subject("refused").text("t").build()); + check(w + "refused drains", Boolean.TRUE, Boolean.valueOf(idle(m))); + Thread.sleep(150); + check(w + "refused is not retried", Integer.valueOf(1), + Integer.valueOf(server.bodies.get())); + check(w + "refused is not delivered", Integer.valueOf(0), + Integer.valueOf(server.count())); + + server.reset(); + server.failData.set(100); + m.send(Mail.to("a@example.com").subject("hopeless").text("t").build()); + check(w + "hopeless drains", Boolean.TRUE, Boolean.valueOf(idle(m))); + check(w + "gives up after max attempts", Integer.valueOf(3), + Integer.valueOf(server.bodies.get())); + if (ds[0] != null) { + check(w + "outbox empty", Long.valueOf(0), Long.valueOf(rows(ds[0], + "cn1_mail_outbox"))); + check(w + "dead rows", Long.valueOf(2), Long.valueOf(rows(ds[0], + "cn1_mail_dead"))); + } + } finally { + m.close(); + if (ds[0] != null) { + ds[0].close(); + } + } + } + + private static void clear(DataSource ds) { + try { + ds.execute("DELETE FROM cn1_mail_outbox", new Object[0]); + ds.execute("DELETE FROM cn1_mail_dead", new Object[0]); + } catch (IOException noTableYet) { + // created by the first send + } + } + + /// The outbox: a mail belongs to the transaction that sent it, and one a + /// stopped server left behind is sent by the next. + private static void outbox(Server server, String database) throws Exception { + String w = database.indexOf("://") > 0 + ? database.substring(0, database.indexOf(':')) + " outbox " : "sqlite outbox "; + server.reset(); + DataSource[] ds = new DataSource[1]; + Mailer m = mailer(server, database, ds); + final Mailer mm = m; + try { + clear(ds[0]); + try { + Transactions.execute(new Transactions.Work() { + public Object run() throws Exception { + mm.send(Mail.to("a@example.com").subject("rolled back").text("t").build()); + throw new IllegalStateException("no"); + } + }); + } catch (IllegalStateException expected) { + // the point + } + final AtomicInteger during = new AtomicInteger(-1); + final Server s = server; + Transactions.execute(new Transactions.Work() { + public Object run() throws Exception { + mm.send(Mail.to("a@example.com").subject("committed").text("t").build()); + Thread.sleep(300); + during.set(s.count()); + return null; + } + }); + check(w + "commit sends", Boolean.TRUE, Boolean.valueOf(await(server, 1))); + check(w + "nothing before the commit", Integer.valueOf(0), + Integer.valueOf(during.get())); + Thread.sleep(400); + check(w + "rollback sends nothing", "committed", server.subjects()); + check(w + "drains", Boolean.TRUE, Boolean.valueOf(idle(m))); + check(w + "table empty", Long.valueOf(0), Long.valueOf(rows(ds[0], + "cn1_mail_outbox"))); + // A row another server's sweep took while this one was getting to + // it is that server's to send: here its time is moved, as a claim + // moves it, before the commit hands the mail to this process. + final DataSource d = ds[0]; + Transactions.execute(new Transactions.Work() { + public Object run() throws Exception { + mm.send(Mail.to("a@example.com").subject("claimed").text("t").build()); + d.execute("UPDATE cn1_mail_outbox SET next_attempt = next_attempt + 3600000", + new Object[0]); + return null; + } + }); + check(w + "claimed row let go", Boolean.TRUE, Boolean.valueOf(idle(m))); + Thread.sleep(300); + check(w + "a row another server claimed is not sent here", "committed", + server.subjects()); + check(w + "and stays for that server", Long.valueOf(1), Long.valueOf(rows(ds[0], + "cn1_mail_outbox"))); + clear(ds[0]); + } finally { + m.close(); + } + // A server that stops with mail it could not send leaves it for the next. + server.reset(); + server.failData.set(100); + m = mailer(server, database, ds, "cn1.mail.retry.backoff", "600", + "cn1.mail.retry.max-attempts", "5"); + try { + m.send(Mail.to("a@example.com").subject("left behind").text("t").build()); + long end = System.currentTimeMillis() + 5000; + while (server.bodies.get() < 1 && System.currentTimeMillis() < end) { + Thread.sleep(10); + } + } finally { + m.close(); + ds[0].close(); + } + // A server on the same database with no mail server of its own -- a + // developer's, a test run -- leaves the row alone. + DataSource shared = DataSource.open(database, 2); + Properties none = new Properties(); + none.setProperty("cn1.mail.outbox.sweep", "100"); + Mailer silent = Mailers.open(Config.of(none, "test"), shared); + try { + Thread.sleep(500); + check(w + "a server that sends nothing takes no row", Long.valueOf(1), + Long.valueOf(rows(shared, "cn1_mail_outbox"))); + } finally { + silent.close(); + shared.close(); + } + server.reset(); + m = mailer(server, database, ds); + try { + check(w + "left row", Long.valueOf(1), Long.valueOf(rows(ds[0], "cn1_mail_outbox"))); + check(w + "next start sends it", Boolean.TRUE, Boolean.valueOf(await(server, 1))); + check(w + "the same mail", "left behind", server.subjects()); + long end = System.currentTimeMillis() + 5000; + while (rows(ds[0], "cn1_mail_outbox") > 0 && System.currentTimeMillis() < end) { + Thread.sleep(10); + } + check(w + "row removed", Long.valueOf(0), Long.valueOf(rows(ds[0], + "cn1_mail_outbox"))); + } finally { + m.close(); + ds[0].close(); + } + } + + /// One mail of every shape through a real server. + private static void real(String url) throws Exception { + Properties p = new Properties(); + p.setProperty("cn1.mail.url", url); + p.setProperty("cn1.mail.from", env("CN1_MAILCHECK_FROM", "Mail Check ")); + String tls = System.getenv("CN1_MAILCHECK_STARTTLS"); + if (tls != null) { + p.setProperty("cn1.mail.smtp.starttls", tls); + } + String ca = System.getenv("CN1_MAILCHECK_CA"); + if (ca != null) { + p.setProperty("cn1.mail.smtp.ca", ca); + } + String to = env("CN1_MAILCHECK_TO", "someone@example.com"); + Mailer m = Mailers.open(Config.of(p, "test"), null); + try { + byte[] image = new byte[3000]; + for (int i = 0; i < image.length; i++) { + image[i] = (byte) (i * 31 + 7); + } + m.sendNow(Mail.to(to).subject("Plain").text("line one\n.dot\nline three").build()); + m.sendNow(Mail.to("Ren" + ACUTE + "e <" + to + ">") + .subject("Caf" + ACUTE + " " + ACUTE + "t" + ACUTE + " and a rather long " + + "subject that has to be folded over more than one line of header") + .text("d" + ACUTE + "j" + (char) 0xe0 + " vu") + .html("

d" + ACUTE + "j" + (char) 0xe0 + " vu

") + .inline("logo", "image/png", image) + .attach("r" + ACUTE + "sum" + ACUTE + ".bin", "application/octet-stream", image) + .build()); + check("real server took both", Boolean.TRUE, Boolean.TRUE); + } catch (IOException err) { + check("real server", "sent", String.valueOf(err)); + } finally { + m.close(); + } + } + + private static String env(String name, String fallback) { + String v = System.getenv(name); + return v == null || v.length() == 0 ? fallback : v; + } + + private static void check(String what, Object expected, Object actual) { + if (expected == null ? actual == null : expected.equals(actual)) { + passed++; + } else { + failures.add(what + ": expected " + expected + " but was " + actual); + } + } + + // ------------------------------------------------------- reading a mail + + private static int hexDigit(char c) { + return c >= '0' && c <= '9' ? c - '0' : c >= 'A' && c <= 'F' ? c - 'A' + 10 + : c >= 'a' && c <= 'f' ? c - 'a' + 10 : -1; + } + + /// Decodes a quoted-printable body. + static byte[] quotedPrintable(String s) { + byte[] out = new byte[s.length()]; + int n = 0; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '=' && i + 2 < s.length() && s.charAt(i + 1) == '\r' + && s.charAt(i + 2) == '\n') { + i += 2; + } else if (c == '=' && i + 2 < s.length()) { + out[n++] = (byte) (hexDigit(s.charAt(i + 1)) * 16 + hexDigit(s.charAt(i + 2))); + i += 2; + } else { + out[n++] = (byte) c; + } + } + byte[] exact = new byte[n]; + System.arraycopy(out, 0, exact, 0, n); + return exact; + } + + /// Decodes the `=?UTF-8?B?...?=` words of a header value; white space + /// between two of them is not part of the text. + static String decodeWords(String value) throws IOException { + if (value == null) { + return null; + } + StringBuilder out = new StringBuilder(); + byte[] carried = new byte[0]; + int i = 0; + boolean lastWasWord = false; + while (i < value.length()) { + int start = value.indexOf("=?UTF-8?B?", i); + if (start < 0) { + break; + } + int end = value.indexOf("?=", start + 10); + String between = value.substring(i, start); + if (!(lastWasWord && between.trim().length() == 0)) { + out.append(WireBuffer.fromUtf8(carried)); + carried = new byte[0]; + out.append(between); + } + byte[] piece = Base64.decode(value.substring(start + 10, end)); + byte[] joined = new byte[carried.length + piece.length]; + System.arraycopy(carried, 0, joined, 0, carried.length); + System.arraycopy(piece, 0, joined, carried.length, piece.length); + carried = joined; + lastWasWord = true; + i = end + 2; + } + out.append(WireBuffer.fromUtf8(carried)); + out.append(value.substring(i)); + return out.toString(); + } + + /// One part of a MIME message. + static final class Part { + String head = ""; + String type = ""; + String content = ""; + final List parts = new ArrayList(); + + static Part parse(String data) { + Part p = new Part(); + int split = data.indexOf("\r\n\r\n"); + p.head = split < 0 ? data : data.substring(0, split + 2); + p.content = split < 0 ? "" : data.substring(split + 4); + String t = p.header("Content-Type"); + p.type = t == null ? "text/plain" : t; + int b = p.type.indexOf("boundary=\""); + if (b >= 0) { + String boundary = "--" + p.type.substring(b + 10, p.type.indexOf('"', b + 10)); + int at = p.content.indexOf(boundary); + while (at >= 0) { + int from = at + boundary.length(); + if (p.content.startsWith("--", from)) { + break; + } + from += 2; + int next = p.content.indexOf("\r\n" + boundary, from); + if (next < 0) { + break; + } + p.parts.add(parse(p.content.substring(from, next))); + at = next + 2; + } + } + return p; + } + + String header(String name) { + return headerOf(head, name, true); + } + + byte[] bytes() throws IOException { + String e = header("Content-Transfer-Encoding"); + if ("base64".equalsIgnoreCase(e)) { + StringBuilder sb = new StringBuilder(content.length()); + for (int i = 0; i < content.length(); i++) { + char c = content.charAt(i); + if (c != '\r' && c != '\n') { + sb.append(c); + } + } + return Base64.decode(sb.toString()); + } + if ("quoted-printable".equalsIgnoreCase(e)) { + return quotedPrintable(content); + } + return WireBuffer.utf8(content); + } + + String text() throws IOException { + return WireBuffer.fromUtf8(bytes()); + } + } + + /// The value of a header, unfolded when asked. + static String headerOf(String head, String name, boolean unfold) { + String[] lines = split(head); + for (int i = 0; i < lines.length; i++) { + String line = lines[i]; + if (line.length() > name.length() && line.charAt(name.length()) == ':' + && line.regionMatches(true, 0, name, 0, name.length())) { + StringBuilder sb = new StringBuilder(line.substring(name.length() + 1).trim()); + while (i + 1 < lines.length && lines[i + 1].length() > 0 + && (lines[i + 1].charAt(0) == ' ' || lines[i + 1].charAt(0) == '\t')) { + i++; + sb.append(unfold ? "" : "\r\n").append(unfold ? " " + lines[i].trim() + : lines[i]); + } + return sb.toString(); + } + } + return null; + } + + static String[] split(String text) { + List lines = new ArrayList(); + int from = 0; + while (true) { + int nl = text.indexOf("\r\n", from); + if (nl < 0) { + if (from < text.length()) { + lines.add(text.substring(from)); + } + break; + } + lines.add(text.substring(from, nl)); + from = nl + 2; + } + return lines.toArray(new String[lines.size()]); + } + + /// A mail the server took. + static final class Received { + String from; + final List recipients = new ArrayList(); + /// The message as sent, dots taken off, each byte one char. + String data; + + String header(String name) { + int split = data.indexOf("\r\n\r\n"); + return headerOf(split < 0 ? data : data.substring(0, split + 2), name, true); + } + + String rawHeader(String name) { + int split = data.indexOf("\r\n\r\n"); + return headerOf(split < 0 ? data : data.substring(0, split + 2), name, false); + } + + String body() { + int split = data.indexOf("\r\n\r\n"); + return split < 0 ? "" : data.substring(split + 4); + } + + /// What a strict server would object to: empty when nothing. + String wireProblems() { + StringBuilder sb = new StringBuilder(); + String[] lines = split(data); + boolean head = true; + for (String line : lines) { + if (line.length() == 0) { + head = false; + } + if (line.length() > (head ? 78 : 76) && !"7bit".equals( + header("Content-Transfer-Encoding"))) { + sb.append("long line (").append(line.length()).append("): ").append(line) + .append('\n'); + } + for (int i = 0; i < line.length(); i++) { + char c = line.charAt(i); + if (c > 126 || c < 32 && c != '\t') { + sb.append("byte ").append((int) c).append(" in: ").append(line) + .append('\n'); + break; + } + } + } + return sb.toString(); + } + } + + // ------------------------------------------------------ the mail server + + /// An SMTP server of a few dozen lines: it takes mail, records it, and + /// misbehaves on request. + static final class Server implements Runnable { + final ServerSocket socket; + final int port; + final AtomicBoolean stopped = new AtomicBoolean(); + final List received = new ArrayList(); + final AtomicInteger connections = new AtomicInteger(); + final AtomicInteger bodies = new AtomicInteger(); + final AtomicInteger failData = new AtomicInteger(); + final AtomicInteger rejectData = new AtomicInteger(); + final AtomicBoolean pipelined = new AtomicBoolean(); + final AtomicBoolean sawSize = new AtomicBoolean(); + final AtomicBoolean sawUtf8 = new AtomicBoolean(); + final AtomicBoolean sawAuth = new AtomicBoolean(); + boolean pipelining = true; + boolean startTls; + boolean inject; + boolean smtpUtf8; + long maxSize; + + Server() throws IOException { + socket = ServerSocket.bind("127.0.0.1", 0, 16); + port = socket.getPort(); + Thread t = new Thread(this, "fake-smtp"); + t.setDaemon(true); + t.start(); + } + + synchronized void reset() { + received.clear(); + connections.set(0); + bodies.set(0); + failData.set(0); + rejectData.set(0); + pipelined.set(false); + sawSize.set(false); + sawUtf8.set(false); + sawAuth.set(false); + pipelining = true; + startTls = false; + inject = false; + smtpUtf8 = false; + maxSize = 0; + } + + synchronized int count() { + return received.size(); + } + + synchronized Received last() { + return received.get(received.size() - 1); + } + + synchronized String subjects() { + StringBuilder sb = new StringBuilder(); + for (Received r : received) { + sb.append(sb.length() == 0 ? "" : " ").append(r.header("Subject")); + } + return sb.toString(); + } + + void stop() { + stopped.set(true); + socket.close(); + } + + public void run() { + while (!stopped.get()) { + final int fd = socket.accept(); + if (fd < 0) { + try { + Thread.sleep(5); + } catch (InterruptedException err) { + return; + } + continue; + } + connections.incrementAndGet(); + Thread t = new Thread(new Runnable() { + public void run() { + try { + ServerSocket.setBlocking(fd, true); + ServerSocket.setTimeout(fd, 20000); + serve(fd); + } catch (IOException | RuntimeException err) { + // The client went away; the checks say what was missing. + } finally { + ServerSocket.closeFd(fd); + } + } + }, "fake-smtp-conn"); + t.setDaemon(true); + t.start(); + } + } + + private final class Reader { + final int fd; + final byte[] buffer = new byte[65536]; + int position; + int limit; + + Reader(int fd) { + this.fd = fd; + } + + /// A line without its CRLF, each byte one char; null at the end. + String line() throws IOException { + StringBuilder sb = new StringBuilder(); + while (true) { + if (position >= limit) { + int n = ServerSocket.read(fd, buffer, 0, buffer.length); + if (n <= 0) { + return null; + } + position = 0; + limit = n; + } + char c = (char) (buffer[position++] & 0xff); + if (c == '\n') { + int len = sb.length(); + if (len > 0 && sb.charAt(len - 1) == '\r') { + sb.setLength(len - 1); + } + return sb.toString(); + } + sb.append(c); + } + } + + boolean more() { + return position < limit; + } + } + + private void say(int fd, String text) throws IOException { + byte[] b = WireBuffer.ascii(text + "\r\n"); + ServerSocket.write(fd, b, 0, b.length); + } + + /// What the client sent as bytes-in-chars, as the text it was. + private String utf8(String latin) throws IOException { + byte[] b = new byte[latin.length()]; + for (int i = 0; i < b.length; i++) { + b[i] = (byte) latin.charAt(i); + } + return WireBuffer.fromUtf8(b); + } + + private void serve(int fd) throws IOException { + Reader in = new Reader(fd); + say(fd, "220 fake.example ESMTP"); + Received current = null; + while (true) { + String line = in.line(); + if (line == null) { + return; + } + if (line.regionMatches(true, 0, "EHLO", 0, 4)) { + StringBuilder sb = new StringBuilder("250-fake.example"); + if (pipelining) { + sb.append("\r\n250-PIPELINING"); + } + if (startTls) { + sb.append("\r\n250-STARTTLS"); + } + if (smtpUtf8) { + sb.append("\r\n250-SMTPUTF8"); + } + sb.append("\r\n250-AUTH PLAIN LOGIN"); + sb.append("\r\n250 SIZE ").append(maxSize); + say(fd, sb.toString()); + } else if (line.regionMatches(true, 0, "STARTTLS", 0, 8)) { + // No TLS here: what is checked is that the client refuses + // what arrives with the go-ahead. + say(fd, inject ? "220 go ahead\r\n250 injected" : "220 go ahead"); + in.line(); + return; + } else if (line.regionMatches(true, 0, "AUTH", 0, 4)) { + sawAuth.set(true); + say(fd, "235 ok"); + } else if (line.regionMatches(true, 0, "MAIL FROM:", 0, 10)) { + if (in.more()) { + pipelined.set(true); + } + if (line.indexOf(" SIZE=") > 0) { + sawSize.set(true); + } + if (line.indexOf(" SMTPUTF8") > 0) { + sawUtf8.set(true); + } + current = new Received(); + current.from = utf8(line.substring(line.indexOf('<') + 1, line.indexOf('>'))); + say(fd, "250 ok"); + } else if (line.regionMatches(true, 0, "RCPT TO:", 0, 8)) { + String to = utf8(line.substring(line.indexOf('<') + 1, line.indexOf('>'))); + if (current == null) { + say(fd, "503 no sender"); + } else if (to.startsWith("reject")) { + say(fd, "550 no such user"); + } else if (to.startsWith("defer")) { + say(fd, "451 try later"); + } else { + current.recipients.add(to); + say(fd, "250 ok"); + } + } else if (line.regionMatches(true, 0, "DATA", 0, 4)) { + if (current == null || current.recipients.isEmpty()) { + say(fd, "554 no valid recipients"); + continue; + } + say(fd, "354 go ahead"); + StringBuilder data = new StringBuilder(); + while (true) { + String l = in.line(); + if (l == null) { + return; + } + if (".".equals(l)) { + break; + } + data.append(l.startsWith(".") ? l.substring(1) : l).append("\r\n"); + } + bodies.incrementAndGet(); + if (rejectData.get() > 0) { + rejectData.decrementAndGet(); + say(fd, "554 content refused"); + } else if (failData.get() > 0) { + failData.decrementAndGet(); + say(fd, "451 try later"); + } else { + current.data = data.toString(); + synchronized (this) { + received.add(current); + } + say(fd, "250 queued"); + } + current = null; + } else if (line.regionMatches(true, 0, "RSET", 0, 4)) { + current = null; + say(fd, "250 ok"); + } else if (line.regionMatches(true, 0, "NOOP", 0, 4)) { + say(fd, "250 ok"); + } else if (line.regionMatches(true, 0, "QUIT", 0, 4)) { + say(fd, "221 bye"); + return; + } else { + say(fd, "500 what"); + } + } + } + } +} diff --git a/vm/backend/demo/mailcheck/com/demo/MailFlows.java b/vm/backend/demo/mailcheck/com/demo/MailFlows.java new file mode 100644 index 00000000000..ba206d04fcc --- /dev/null +++ b/vm/backend/demo/mailcheck/com/demo/MailFlows.java @@ -0,0 +1,366 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import com.codename1.backend.DataSource; +import com.codename1.backend.Migrations; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.security.Clock; +import com.codename1.backend.security.SecuritySchema; +import com.codename1.backend.security.core.userdetails.InMemoryUserDetailsManager; +import com.codename1.backend.security.core.userdetails.User; +import com.codename1.backend.security.crypto.PasswordEncoder; +import com.codename1.backend.security.crypto.Pbkdf2Sha256PasswordEncoder; +import com.codename1.backend.security.mail.EmailVerificationService; +import com.codename1.backend.security.mail.InMemoryOneTimeTokenRepository; +import com.codename1.backend.security.mail.InMemoryVerifiedEmailRepository; +import com.codename1.backend.security.mail.JdbcOneTimeTokenRepository; +import com.codename1.backend.security.mail.JdbcVerifiedEmailRepository; +import com.codename1.backend.security.mail.OneTimeTokenRepository; +import com.codename1.backend.security.mail.OneTimeTokenService; +import com.codename1.backend.security.mail.PasswordResetService; +import com.codename1.backend.security.mail.VerifiedEmailRepository; +import com.codename1.backend.security.mfa.EmailCodeRepository; +import com.codename1.backend.security.mfa.EmailCodeService; +import com.codename1.backend.security.mfa.InMemoryEmailCodeRepository; +import com.codename1.backend.security.mfa.JdbcEmailCodeRepository; +import java.util.ArrayList; +import java.util.List; + +/// The flows the security layer runs over mail -- one-time tokens, a password +/// reset, proving an address, a mailed sign-in code -- against the transport +/// that keeps mail in memory, with the stores in memory and in a database. +final class MailFlows { + private final List failures; + private int passed; + private final MailSender mail = new MailSender(); + private final long[] now = {1000000000000L}; + private final Clock clock = new Clock() { + @Override + public long currentTimeMillis() { + return now[0]; + } + }; + + private MailFlows(List failures) { + this.failures = failures; + } + + /// Runs every check; answers how many passed and adds the rest to + /// `failures`. + /// + /// @param database a database URL to keep the stores in, or null for the + /// ones in memory + static int run(String database, List failures) throws Exception { + // What the flows mail from; read by the mail support of this process, + // which has no server and so no application.properties. Set for these + // checks alone: it would be every other mailer's sender too. + System.setProperty("cn1.mail.from", "Accounts "); + try { + return stores(database, failures); + } finally { + System.clearProperty("cn1.mail.from"); + } + } + + private static int stores(String database, List failures) throws Exception { + MailFlows f = new MailFlows(failures); + if (database == null) { + f.all("memory ", new InMemoryOneTimeTokenRepository(), + new InMemoryVerifiedEmailRepository(), new InMemoryEmailCodeRepository()); + return f.passed; + } + DataSource ds = DataSource.open(database, 2); + try { + SecuritySchema.resetTableNames(); + Migrations.of(ds, SecuritySchema.migrations()).migrate(); + String w = database.indexOf("://") > 0 + ? database.substring(0, database.indexOf(':')) + " " : "sqlite "; + String[] tables = {"cn1_one_time_token", "cn1_verified_email", "cn1_mfa_email"}; + for (String table : tables) { + ds.execute("DELETE FROM " + table, new Object[0]); + } + f.all(w, new JdbcOneTimeTokenRepository(ds), new JdbcVerifiedEmailRepository(ds), + new JdbcEmailCodeRepository(ds)); + } finally { + ds.close(); + } + return f.passed; + } + + private void check(String what, Object expected, Object actual) { + if (expected == null ? actual == null : expected.equals(actual)) { + passed++; + } else { + failures.add(what + ": expected " + expected + " but was " + actual); + } + } + + private void all(String w, OneTimeTokenRepository tokenStore, + VerifiedEmailRepository addressStore, EmailCodeRepository codeStore) + throws Exception { + OneTimeTokenService tokens = new OneTimeTokenService(tokenStore).clock(clock); + tokens(w, tokens); + reset(w, tokens); + EmailVerificationService verification = verification(w, tokens, addressStore); + codes(w, codeStore, verification); + } + + private void tokens(String w, OneTimeTokenService tokens) { + String a = tokens.create("p", "Ada@example.com", 60); + check(w + "token length", Integer.valueOf(43), Integer.valueOf(a.length())); + check(w + "token wrong purpose", null, tokens.consume("q", a)); + check(w + "token survives a wrong purpose", "Ada@example.com", tokens.consume("p", a)); + check(w + "token works once", null, tokens.consume("p", a)); + String b = tokens.create("p", "ada@example.com", 60); + String c = tokens.create("p", "ADA@example.com", 60); + check(w + "a newer token replaces the older", null, tokens.consume("p", b)); + check(w + "tokens differ", Boolean.FALSE, Boolean.valueOf(b.equals(c))); + String other = tokens.create("q", "ada@example.com", 60); + check(w + "another purpose is another token", "ADA@example.com", tokens.consume("p", c)); + check(w + "and is still good", "ada@example.com", tokens.consume("q", other)); + String d = tokens.create("p", "ada@example.com", 60); + now[0] += 61000; + check(w + "token expires", null, tokens.consume("p", d)); + String e = tokens.create("p", "ada@example.com", 60); + tokens.revoke("p", "ADA@EXAMPLE.COM"); + check(w + "token revoked", null, tokens.consume("p", e)); + check(w + "garbage token", null, tokens.consume("p", "nope")); + check(w + "null token", null, tokens.consume("p", null)); + } + + private static String linkToken(Mail m) { + String text = m.getText(); + int at = text.indexOf("token="); + return at < 0 ? null : text.substring(at + 6, at + 6 + 43); + } + + private void reset(String w, OneTimeTokenService tokens) throws Exception { + PasswordEncoder encoder = new Pbkdf2Sha256PasswordEncoder(); + InMemoryUserDetailsManager users = new InMemoryUserDetailsManager( + new User("ada@example.com", encoder.encode("old password"), + new ArrayList()), + new User("bob", encoder.encode("old password"), + new ArrayList()), + new User("off@example.com", encoder.encode("old password"), false, true, true, + true, new ArrayList())); + PasswordResetService service = new PasswordResetService(tokens, mail, users, users, + encoder); + mail.clearSent(); + service.request("nobody@example.com", "https://example.com/reset-password"); + service.request("bob", "https://example.com/reset-password"); + service.request("off@example.com", "https://example.com/reset-password"); + service.request(null, "https://example.com/reset-password"); + check(w + "reset: nothing for no user, no address or a disabled account", + Integer.valueOf(0), Integer.valueOf(mail.sent().size())); + service.request("ADA@example.com", "https://example.com/reset-password"); + check(w + "reset: one mail", Integer.valueOf(1), Integer.valueOf(mail.sent().size())); + Mail m = mail.sent().get(0); + check(w + "reset: to the user", "[ada@example.com]", String.valueOf(m.getTo())); + check(w + "reset: subject", "Reset your password", m.getSubject()); + check(w + "reset: link in the text", Boolean.TRUE, Boolean.valueOf( + m.getText().indexOf("https://example.com/reset-password?token=") > 0)); + check(w + "reset: link in the html", Boolean.TRUE, Boolean.valueOf( + m.getHtml().indexOf("href=\"https://example.com/reset-password?token=") > 0)); + String token = linkToken(m); + check(w + "reset: short password refused", Boolean.FALSE, + Boolean.valueOf(service.reset(token, "short"))); + check(w + "reset: a refused password keeps the link", Boolean.TRUE, + Boolean.valueOf(service.reset(token, "a new password"))); + check(w + "reset: password changed", Boolean.TRUE, Boolean.valueOf(encoder.matches( + "a new password", users.loadUserByUsername("ada@example.com").getPassword()))); + check(w + "reset: link works once", Boolean.FALSE, + Boolean.valueOf(service.reset(token, "another password"))); + check(w + "reset: password unchanged by a used link", Boolean.TRUE, + Boolean.valueOf(encoder.matches("a new password", + users.loadUserByUsername("ada@example.com").getPassword()))); + mail.clearSent(); + service.request("ada@example.com", "https://example.com/reset-password"); + String first = linkToken(mail.sent().get(0)); + service.request("ada@example.com", "https://example.com/reset-password"); + String second = linkToken(mail.sent().get(1)); + check(w + "reset: an older link stops working", Boolean.FALSE, + Boolean.valueOf(service.reset(first, "another password"))); + now[0] += 31 * 60000L; + check(w + "reset: link expires", Boolean.FALSE, + Boolean.valueOf(service.reset(second, "another password"))); + boolean threw = false; + try { + service.request("ada@example.com", "/reset-password"); + } catch (IllegalArgumentException expected) { + threw = true; + } + check(w + "reset: a path is not an address to mail", Boolean.TRUE, + Boolean.valueOf(threw)); + mail.clearSent(); + } + + private EmailVerificationService verification(String w, OneTimeTokenService tokens, + VerifiedEmailRepository store) throws Exception { + EmailVerificationService service = new EmailVerificationService(tokens, mail, store) + .clock(clock); + mail.clearSent(); + check(w + "verify: unknown user", Boolean.FALSE, + Boolean.valueOf(service.isVerified("bob"))); + check(w + "verify: user name that is an address", "x@example.com", + service.emailFor("x@example.com")); + check(w + "verify: no address for a plain name", null, service.emailFor("bob")); + service.send("Bob", "bob@example.com", "https://example.com/verify-email"); + check(w + "verify: one mail", Integer.valueOf(1), Integer.valueOf(mail.sent().size())); + check(w + "verify: to the address", "[bob@example.com]", + String.valueOf(mail.sent().get(0).getTo())); + check(w + "verify: not yet", Boolean.FALSE, Boolean.valueOf(service.isVerified("bob"))); + check(w + "verify: an unproved address is not used", null, service.emailFor("bob")); + String token = linkToken(mail.sent().get(0)); + check(w + "verify: wrong token", null, service.confirm(token.substring(1) + "A")); + check(w + "verify: confirm", "Bob", service.confirm(token)); + check(w + "verify: proved", Boolean.TRUE, Boolean.valueOf(service.isVerified("BOB"))); + check(w + "verify: proved address", "bob@example.com", service.emailFor("bob")); + check(w + "verify: link works once", null, service.confirm(token)); + service.send("bob", "robert@example.com", "https://example.com/verify-email"); + check(w + "verify: a new address is unproved", Boolean.FALSE, + Boolean.valueOf(service.isVerified("bob"))); + check(w + "verify: confirm the new one", "bob", + service.confirm(linkToken(mail.sent().get(1)))); + check(w + "verify: new address", "robert@example.com", service.getVerifiedEmail("bob")); + boolean threw = false; + try { + service.send("bob", "not an address", "https://example.com/verify-email"); + } catch (IllegalArgumentException expected) { + threw = true; + } + check(w + "verify: bad address refused", Boolean.TRUE, Boolean.valueOf(threw)); + check(w + "verify: a refused address changes nothing", "robert@example.com", + service.getVerifiedEmail("bob")); + // An address is proved by the link mailed to it and by no other. + store.savePending("Carol", "first@example.com", "proof-of-the-first"); + store.savePending("carol", "second@example.com", "proof-of-the-second"); + check(w + "verify: a link to an earlier address proves nothing", Boolean.FALSE, + Boolean.valueOf(store.markVerified("carol", "proof-of-the-first", 5))); + check(w + "verify: still unproved", Boolean.FALSE, + Boolean.valueOf(store.isVerified("carol"))); + check(w + "verify: the link to this address proves it", Boolean.TRUE, + Boolean.valueOf(store.markVerified("CAROL", "proof-of-the-second", 5))); + check(w + "verify: and proves it once", Boolean.FALSE, + Boolean.valueOf(store.markVerified("carol", "proof-of-the-second", 6))); + check(w + "verify: the address proved", "second@example.com", + store.isVerified("carol") ? store.findEmail("carol") : null); + store.delete("carol"); + mail.clearSent(); + return service; + } + + private static String mailedCode(Mail m) { + String text = m.getText(); + int at = text.indexOf("code is "); + return text.substring(at + 8, at + 14); + } + + private static String wrong(String code) { + return (code.charAt(0) == '9' ? "0" : "9") + code.substring(1); + } + + private void codes(String w, EmailCodeRepository store, EmailVerificationService addresses) + throws Exception { + EmailCodeService service = new EmailCodeService(store, mail).clock(clock) + .addressResolver(addresses); + mail.clearSent(); + check(w + "code: off by default", Boolean.FALSE, + Boolean.valueOf(service.isEnabled("bob"))); + check(w + "code: nothing to verify", Boolean.FALSE, + Boolean.valueOf(service.verify("bob", "123456"))); + service.enable("Bob"); + check(w + "code: enabled", Boolean.TRUE, Boolean.valueOf(service.isEnabled("bob"))); + check(w + "code: sent", Boolean.TRUE, Boolean.valueOf(service.send("bob"))); + check(w + "code: to the proved address", "[robert@example.com]", + String.valueOf(mail.sent().get(0).getTo())); + String code = mailedCode(mail.sent().get(0)); + check(w + "code: six digits", Boolean.TRUE, + Boolean.valueOf(EmailCodeService.isCodeShaped(code))); + check(w + "code: not resent at once", Boolean.FALSE, Boolean.valueOf(service.send("bob"))); + check(w + "code: still one mail", Integer.valueOf(1), + Integer.valueOf(mail.sent().size())); + check(w + "code: wrong", Boolean.FALSE, + Boolean.valueOf(service.verify("bob", wrong(code)))); + check(w + "code: not a code", Boolean.FALSE, + Boolean.valueOf(service.verify("bob", "abcdef"))); + check(w + "code: right, written with a space", Boolean.TRUE, Boolean.valueOf( + service.verify("BOB", code.substring(0, 3) + " " + code.substring(3)))); + check(w + "code: works once", Boolean.FALSE, Boolean.valueOf(service.verify("bob", code))); + check(w + "code: still enabled", Boolean.TRUE, Boolean.valueOf(service.isEnabled("bob"))); + + // No wait: the code before it was used to sign in. + check(w + "code: mailed at once after a sign-in", Boolean.TRUE, + Boolean.valueOf(service.send("bob"))); + code = mailedCode(mail.sent().get(1)); + service.verify("bob", wrong(code)); + service.verify("bob", wrong(code)); + now[0] += 31000; + check(w + "code: another one", Boolean.TRUE, Boolean.valueOf(service.send("bob"))); + String older = code; + code = mailedCode(mail.sent().get(2)); + check(w + "code: the one before it is gone", Boolean.TRUE, + Boolean.valueOf(older.equals(code) || !service.verify("bob", older))); + // Three were left, whichever code they are spent on; one went above + // unless the two codes happened to be the same. + for (int i = 0; i < 3; i++) { + service.verify("bob", wrong(code)); + } + check(w + "code: asking for another is not a way to more tries", Boolean.FALSE, + Boolean.valueOf(service.verify("bob", code))); + now[0] += 31000; + check(w + "code: none is mailed while the tries are used up", Boolean.FALSE, + Boolean.valueOf(service.send("bob"))); + check(w + "code: still three mails", Integer.valueOf(3), + Integer.valueOf(mail.sent().size())); + // The wait ends when the last code runs out, however often one was + // asked for meanwhile. + now[0] += 570000; + check(w + "code: mailed again once the last one ran out", Boolean.TRUE, + Boolean.valueOf(service.send("bob"))); + code = mailedCode(mail.sent().get(3)); + check(w + "code: and the tries start again", Boolean.TRUE, + Boolean.valueOf(service.verify("bob", code))); + now[0] += 601000; + service.send("bob"); + code = mailedCode(mail.sent().get(4)); + check(w + "code: a later code works", Boolean.TRUE, Boolean.valueOf(service.verify("bob", code))); + now[0] += 31000; + service.send("bob"); + code = mailedCode(mail.sent().get(5)); + now[0] += 601000; + check(w + "code: expires", Boolean.FALSE, Boolean.valueOf(service.verify("bob", code))); + now[0] += 1000; + service.send("bob"); + code = mailedCode(mail.sent().get(6)); + service.disable("bob"); + check(w + "code: disabled", Boolean.FALSE, Boolean.valueOf(service.isEnabled("bob"))); + check(w + "code: a waiting code goes with it", Boolean.FALSE, + Boolean.valueOf(service.verify("bob", code))); + service.enable("nobody"); + check(w + "code: no address, nothing sent", Boolean.FALSE, + Boolean.valueOf(service.send("nobody"))); + mail.clearSent(); + } +} diff --git a/vm/backend/demo/mailcheck/com/demo/SesCheck.java b/vm/backend/demo/mailcheck/com/demo/SesCheck.java new file mode 100644 index 00000000000..a70090556ff --- /dev/null +++ b/vm/backend/demo/mailcheck/com/demo/SesCheck.java @@ -0,0 +1,289 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.demo; + +import com.codename1.backend.Base64; +import com.codename1.backend.ServerSocket; +import com.codename1.backend.aws.Credentials; +import com.codename1.backend.mail.MailAuthenticationException; +import com.codename1.backend.mail.MailSendException; +import com.codename1.backend.mail.ses.SesTransport; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/// The SES transport against an endpoint of this process: what it asks for, +/// how it signs, how it splits a long recipient list, and what each refusal +/// becomes. Nothing here reaches Amazon. +final class SesCheck implements Runnable { + private final ServerSocket socket; + private final AtomicBoolean stopped = new AtomicBoolean(); + private final AtomicInteger status = new AtomicInteger(200); + private final List requests = new ArrayList(); + private final List bodies = new ArrayList(); + private final List failures; + private int passed; + + private SesCheck(List failures) throws IOException { + this.failures = failures; + socket = ServerSocket.bind("127.0.0.1", 0, 16); + Thread t = new Thread(this, "fake-ses"); + t.setDaemon(true); + t.start(); + } + + /// Runs the checks; answers how many passed. + static int run(List failures) throws Exception { + SesCheck check = new SesCheck(failures); + try { + check.checks(); + } finally { + check.stopped.set(true); + check.socket.close(); + } + return check.passed; + } + + private void check(String what, Object expected, Object actual) { + if (expected == null ? actual == null : expected.equals(actual)) { + passed++; + } else { + failures.add("ses " + what + ": expected " + expected + " but was " + actual); + } + } + + private synchronized String request(int i) { + return i < requests.size() ? requests.get(i) : ""; + } + + private synchronized String body(int i) { + return i < bodies.size() ? bodies.get(i) : ""; + } + + private synchronized int count() { + return requests.size(); + } + + private synchronized void clear() { + requests.clear(); + bodies.clear(); + } + + private static String header(String request, String name) { + // By hand: the device's String has no split. + int at = request.indexOf("\r\n"); + while (at >= 0) { + int next = request.indexOf("\r\n", at + 2); + String line = request.substring(at + 2, next < 0 ? request.length() : next); + int colon = line.indexOf(':'); + if (colon > 0 && line.substring(0, colon).trim().equalsIgnoreCase(name)) { + return line.substring(colon + 1).trim(); + } + at = next; + } + return null; + } + + private void checks() throws Exception { + Credentials who = new Credentials("AKIDEXAMPLE", "secret-of-the-example", null); + byte[] mime = WireBuffer.ascii("Subject: Hello\r\n\r\nA body\r\n"); + SesTransport ses = new SesTransport("ses://127.0.0.1:" + socket.getPort(), + "eu-west-1", "receipts", who); + check("name", "ses", ses.getName()); + + ses.send("sender@example.com", new String[] {"a@example.com", "b@example.com"}, mime); + check("one request", Integer.valueOf(1), Integer.valueOf(count())); + String request = request(0); + check("request line", "POST /v2/email/outbound-emails HTTP/1.1", + request.substring(0, Math.max(0, request.indexOf("\r\n")))); + String auth = String.valueOf(header(request, "Authorization")); + check("signed for ses in the region", Boolean.TRUE, Boolean.valueOf( + auth.startsWith("AWS4-HMAC-SHA256 Credential=AKIDEXAMPLE/") + && auth.indexOf("/eu-west-1/ses/aws4_request") > 0 + && auth.indexOf("Signature=") > 0)); + check("dated", Boolean.TRUE, Boolean.valueOf(header(request, "X-Amz-Date") != null)); + check("json", "application/json", header(request, "Content-Type")); + String body = body(0); + check("sender", Boolean.TRUE, Boolean.valueOf( + body.indexOf("\"FromEmailAddress\":\"sender@example.com\"") >= 0)); + check("recipients", Boolean.TRUE, Boolean.valueOf( + body.indexOf("\"ToAddresses\":[\"a@example.com\",\"b@example.com\"]") >= 0)); + check("the mail as it was built", Boolean.TRUE, Boolean.valueOf( + body.indexOf("\"Raw\":{\"Data\":\"" + Base64.encode(mime) + "\"}") >= 0)); + check("configuration set", Boolean.TRUE, Boolean.valueOf( + body.indexOf("\"ConfigurationSetName\":\"receipts\"") >= 0)); + check("healthy", Boolean.TRUE, Boolean.valueOf(ses.isHealthy())); + + // More recipients than one request takes. + clear(); + String[] many = new String[120]; + for (int i = 0; i < many.length; i++) { + many[i] = "r" + i + "@example.com"; + } + ses.send("sender@example.com", many, mime); + check("three requests for 120", Integer.valueOf(3), Integer.valueOf(count())); + check("the last holds the rest", Boolean.TRUE, Boolean.valueOf( + body(2).indexOf("[\"r100@example.com\"") > 0 + && body(2).indexOf("\"r119@example.com\"]") > 0)); + + // Refusals. + clear(); + status.set(403); + try { + ses.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("403 is refused", "an exception", "none"); + } catch (MailAuthenticationException err) { + check("403 is the credentials", Integer.valueOf(403), Integer.valueOf(err.getCode())); + } + status.set(400); + try { + ses.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("400 is refused", "an exception", "none"); + } catch (MailSendException err) { + check("400 is permanent", Boolean.TRUE, Boolean.valueOf(err.isPermanent())); + } + status.set(429); + try { + ses.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("429 is refused", "an exception", "none"); + } catch (MailSendException err) { + check("429 is worth another attempt", Boolean.FALSE, + Boolean.valueOf(err.isPermanent())); + } + status.set(503); + try { + ses.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("503 is refused", "an exception", "none"); + } catch (MailSendException err) { + check("503 is worth another attempt", Boolean.FALSE, + Boolean.valueOf(err.isPermanent())); + check("503 is unhealthy", Boolean.FALSE, Boolean.valueOf(ses.isHealthy())); + } + status.set(200); + ses.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("healthy again", Boolean.TRUE, Boolean.valueOf(ses.isHealthy())); + + // No configuration set, no member for one. + clear(); + SesTransport plain = new SesTransport("ses://127.0.0.1:" + socket.getPort(), null, + null, who); + plain.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("no configuration set", Boolean.TRUE, + Boolean.valueOf(body(0).length() > 0 + && body(0).indexOf("ConfigurationSetName") < 0)); + check("default region", Boolean.TRUE, Boolean.valueOf( + String.valueOf(header(request(0), "Authorization")) + .indexOf("/us-east-1/ses/aws4_request") > 0)); + plain.close(); + ses.close(); + try { + ses.send("sender@example.com", new String[] {"a@example.com"}, mime); + check("closed", "an exception", "none"); + } catch (IOException err) { + check("closed", Boolean.TRUE, Boolean.TRUE); + } + try { + new SesTransport("ses://", null, null); + check("no region", "an exception", "none"); + } catch (IOException err) { + check("no region names the fix", Boolean.TRUE, + Boolean.valueOf(err.getMessage().indexOf("ses://eu-west-1") >= 0)); + } + } + + // ------------------------------------------------------------ the server + + public void run() { + while (!stopped.get()) { + final int fd = socket.accept(); + if (fd < 0) { + try { + Thread.sleep(5); + } catch (InterruptedException err) { + return; + } + continue; + } + Thread t = new Thread(new Runnable() { + public void run() { + try { + ServerSocket.setBlocking(fd, true); + ServerSocket.setTimeout(fd, 20000); + serve(fd); + } catch (IOException | RuntimeException err) { + // The client went away; the checks say what was missing. + } finally { + ServerSocket.closeFd(fd); + } + } + }, "fake-ses-conn"); + t.setDaemon(true); + t.start(); + } + } + + /// Requests on one connection, each answered with the status set. + private void serve(int fd) throws IOException { + byte[] buffer = new byte[65536]; + StringBuilder in = new StringBuilder(); + while (true) { + int end; + while ((end = in.indexOf("\r\n\r\n")) < 0) { + int n = ServerSocket.read(fd, buffer, 0, buffer.length); + if (n <= 0) { + return; + } + for (int i = 0; i < n; i++) { + in.append((char) (buffer[i] & 0xff)); + } + } + String head = in.substring(0, end); + String declared = header(head, "Content-Length"); + int length = declared == null ? 0 : Integer.parseInt(declared); + while (in.length() < end + 4 + length) { + int n = ServerSocket.read(fd, buffer, 0, buffer.length); + if (n <= 0) { + return; + } + for (int i = 0; i < n; i++) { + in.append((char) (buffer[i] & 0xff)); + } + } + synchronized (this) { + requests.add(head); + bodies.add(in.substring(end + 4, end + 4 + length)); + } + in.delete(0, end + 4 + length); + int code = status.get(); + String answer = code == 200 ? "{\"MessageId\":\"0100-example\"}" + : "{\"message\":\"refused by the check\"}"; + byte[] out = WireBuffer.ascii("HTTP/1.1 " + code + " Status\r\n" + + "Content-Type: application/json\r\nContent-Length: " + answer.length() + + "\r\n\r\n" + answer); + ServerSocket.write(fd, out, 0, out.length); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/Address.java b/vm/backend/src/com/codename1/backend/mail/Address.java new file mode 100644 index 00000000000..bcbd584658f --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/Address.java @@ -0,0 +1,142 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +/// One mailbox: an optional display name and the address itself. +/// +/// Accepts `user@example.com` and `Display Name `. A quoted +/// local part, a comment and a group are valid RFC 5322 and are refused here: +/// no application sends to one on purpose, and each is a way to smuggle a +/// second recipient past a check that reads the string by eye. +final class Address { + /// The display name, or null. + final String name; + /// The address: `local@domain`. + final String spec; + /// Whether [#spec] is ASCII; one that is not needs the server's SMTPUTF8. + final boolean ascii; + + private Address(String name, String spec, boolean ascii) { + this.name = name; + this.spec = spec; + this.ascii = ascii; + } + + /// Parses one mailbox, or throws IllegalArgumentException naming what is + /// wrong with it. + static Address parse(String value) { + if (value == null) { + throw new IllegalArgumentException("An email address is null"); + } + String v = value.trim(); + String name = null; + String spec = v; + int open = v.lastIndexOf('<'); + if (open >= 0 && v.endsWith(">")) { + spec = v.substring(open + 1, v.length() - 1).trim(); + name = v.substring(0, open).trim(); + if (name.length() >= 2 && name.charAt(0) == '"' + && name.charAt(name.length() - 1) == '"') { + name = name.substring(1, name.length() - 1); + } + if (name.length() == 0) { + name = null; + } + } + if (name != null) { + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (c < ' ' || c == 127) { + throw new IllegalArgumentException("The display name of \"" + printable(value) + + "\" has a control character in it"); + } + } + } + return new Address(name, spec, checkSpec(spec, value)); + } + + /// Whether `spec` is a usable `local@domain`; answers without throwing. + static boolean isValid(String spec) { + if (spec == null) { + return false; + } + try { + checkSpec(spec, spec); + return true; + } catch (IllegalArgumentException err) { + return false; + } + } + + private static boolean checkSpec(String spec, String original) { + int at = spec.indexOf('@'); + if (at <= 0 || at != spec.lastIndexOf('@') || at == spec.length() - 1) { + throw new IllegalArgumentException("\"" + printable(original) + + "\" is not an email address: it needs one @ with text on both sides"); + } + if (spec.length() > 254 || at > 64) { + throw new IllegalArgumentException("\"" + printable(original) + + "\" is longer than an email address may be"); + } + boolean ascii = true; + for (int i = 0; i < spec.length(); i++) { + char c = spec.charAt(i); + if (c <= ' ' || c == 127 || c == '<' || c == '>' || c == '(' || c == ')' + || c == ',' || c == ';' || c == ':' || c == '"' || c == '\\' + || c == '[' || c == ']') { + throw new IllegalArgumentException("\"" + printable(original) + + "\" is not an email address: character " + (int) c + + " cannot be part of one"); + } + if (c > 126) { + ascii = false; + } + } + if (spec.charAt(0) == '.' || spec.charAt(at - 1) == '.' || spec.charAt(at + 1) == '.' + || spec.charAt(spec.length() - 1) == '.' || spec.indexOf("..") >= 0) { + throw new IllegalArgumentException("\"" + printable(original) + + "\" is not an email address: a dot is misplaced"); + } + return ascii; + } + + /// `value` with anything that could break a log line replaced. + private static String printable(String value) { + StringBuilder sb = new StringBuilder(value.length()); + for (int i = 0; i < value.length() && i < 80; i++) { + char c = value.charAt(i); + sb.append(c < ' ' || c == 127 ? '?' : c); + } + return sb.toString(); + } + + /// The domain of the address. + String domain() { + return spec.substring(spec.indexOf('@') + 1); + } + + @Override + public String toString() { + return name == null ? spec : name + " <" + spec + ">"; + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/HtmlText.java b/vm/backend/src/com/codename1/backend/mail/HtmlText.java new file mode 100644 index 00000000000..39b5159f0b8 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/HtmlText.java @@ -0,0 +1,336 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +/// Derives the plain text alternative of an HTML body. +/// +/// Not a renderer: it drops markup, keeps the words, breaks lines where a +/// block ends and writes a link's address after its text -- a password reset +/// mail whose plain part has no link in it is of no use to the reader who +/// sees that part. A mail whose plain text matters is given one with +/// [Mail.Builder#text]. +final class HtmlText { + private HtmlText() { + } + + /// Whether the `<` at `at` begins a tag, a comment or a declaration, as + /// an HTML parser decides it: by the character after it. + private static boolean opensTag(String html, int at) { + if (at + 1 >= html.length()) { + return false; + } + char c = html.charAt(at + 1); + return c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c == '/' || c == '!' || c == '?'; + } + + static String strip(String html) { + StringBuilder out = new StringBuilder(html.length() / 2 + 16); + int n = html.length(); + int i = 0; + String href = null; + int linkStart = 0; + while (i < n) { + char c = html.charAt(i); + if (c == '<' && !opensTag(html, i)) { + // "a < b": a sign, not markup. Read as a tag it would take + // the text after it along, to the next > or to the end. + out.append(c); + i++; + } else if (c == '<') { + int end = html.indexOf('>', i); + if (end < 0) { + // A tag cut short: nothing of it is text. + break; + } + if (html.startsWith("", i + 4); + i = close < 0 ? n : close + 3; + continue; + } + boolean closing = i + 1 < n && html.charAt(i + 1) == '/'; + String tag = tagName(html, closing ? i + 2 : i + 1, end); + if (!closing && ("script".equals(tag) || "style".equals(tag) + || "head".equals(tag) || "title".equals(tag))) { + int close = indexOfClose(html, tag, end); + i = close < 0 ? n : close; + continue; + } + if ("a".equals(tag)) { + if (closing) { + if (href != null) { + String shown = out.substring(linkStart).trim(); + if (!shown.equals(href)) { + out.append(" (").append(href).append(')'); + } + href = null; + } + } else { + href = attribute(html, i, end, "href"); + linkStart = out.length(); + } + } else if ("br".equals(tag) || closing && isBlock(tag)) { + out.append('\n'); + if (closing && ("p".equals(tag) || tag.length() == 2 && tag.charAt(0) == 'h')) { + // A paragraph is set off by a blank line. + out.append('\n'); + } + } else if (!closing && "li".equals(tag)) { + out.append("- "); + } + i = end + 1; + } else if (c == '&') { + i = entity(html, i, out); + } else if (c == '\n' || c == '\r' || c == '\t') { + out.append(' '); + i++; + } else { + out.append(c); + i++; + } + } + return tidy(out); + } + + private static boolean isBlock(String tag) { + return "p".equals(tag) || "div".equals(tag) || "tr".equals(tag) || "li".equals(tag) + || "table".equals(tag) || "ul".equals(tag) || "ol".equals(tag) + || "blockquote".equals(tag) || "section".equals(tag) || "header".equals(tag) + || "footer".equals(tag) || tag.length() == 2 && tag.charAt(0) == 'h' + && tag.charAt(1) >= '1' && tag.charAt(1) <= '6'; + } + + /// The tag's name in lower case; ASCII folded by hand, as a device set to + /// Turkish folds `TITLE` to something that is not `title`. + private static String tagName(String html, int from, int end) { + StringBuilder sb = new StringBuilder(8); + for (int i = from; i < end; i++) { + char c = html.charAt(i); + if (c >= 'A' && c <= 'Z') { + c = (char) (c + 32); + } + if (!(c >= 'a' && c <= 'z' || c >= '0' && c <= '9')) { + break; + } + sb.append(c); + } + return sb.toString(); + } + + private static int indexOfClose(String html, String tag, int from) { + int n = html.length(); + int i = from; + while (true) { + i = html.indexOf("', i); + return end < 0 ? n : end + 1; + } + i += 2; + } + } + + private static String attribute(String html, int from, int end, String name) { + int i = from; + while (i < end) { + int at = indexOfIgnoreCase(html, name, i, end); + if (at < 0) { + return null; + } + int p = at + name.length(); + char before = html.charAt(at - 1); + while (p < end && html.charAt(p) == ' ') { + p++; + } + if (before <= ' ' && p < end && html.charAt(p) == '=') { + p++; + while (p < end && html.charAt(p) == ' ') { + p++; + } + if (p >= end) { + return null; + } + char q = html.charAt(p); + if (q == '"' || q == '\'') { + int close = html.indexOf(q, p + 1); + if (close < 0 || close > end) { + return null; + } + return unescape(html.substring(p + 1, close)); + } + int stop = p; + while (stop < end && html.charAt(stop) > ' ') { + stop++; + } + return unescape(html.substring(p, stop)); + } + i = at + name.length(); + } + return null; + } + + private static int indexOfIgnoreCase(String html, String name, int from, int end) { + for (int i = Math.max(from, 1); i + name.length() <= end; i++) { + if (html.regionMatches(true, i, name, 0, name.length())) { + return i; + } + } + return -1; + } + + private static String unescape(String value) { + if (value.indexOf('&') < 0) { + return value; + } + StringBuilder sb = new StringBuilder(value.length()); + int i = 0; + while (i < value.length()) { + if (value.charAt(i) == '&') { + i = entity(value, i, sb); + } else { + sb.append(value.charAt(i++)); + } + } + return sb.toString(); + } + + /// Appends the character the entity at `i` stands for and answers the + /// index after it; an entity it does not know is copied as it is. + private static int entity(String html, int i, StringBuilder out) { + int semi = html.indexOf(';', i); + if (semi < 0 || semi - i > 9) { + out.append('&'); + return i + 1; + } + String name = html.substring(i + 1, semi); + if ("amp".equals(name)) { + out.append('&'); + } else if ("lt".equals(name)) { + out.append('<'); + } else if ("gt".equals(name)) { + out.append('>'); + } else if ("quot".equals(name)) { + out.append('"'); + } else if ("apos".equals(name)) { + out.append('\''); + } else if ("nbsp".equals(name)) { + out.append(' '); + } else if (name.length() > 1 && name.charAt(0) == '#') { + int code = number(name); + if (code <= 0 || code > 0x10FFFF || code >= 0xD800 && code <= 0xDFFF) { + out.append('&'); + return i + 1; + } + if (code >= 0x10000) { + int v = code - 0x10000; + out.append((char) (0xD800 + (v >> 10))).append((char) (0xDC00 + (v & 0x3FF))); + } else { + out.append((char) code); + } + } else { + out.append('&'); + return i + 1; + } + return semi + 1; + } + + private static int number(String name) { + boolean hex = name.length() > 2 && (name.charAt(1) == 'x' || name.charAt(1) == 'X'); + int value = 0; + for (int i = hex ? 2 : 1; i < name.length(); i++) { + char c = name.charAt(i); + int d; + if (c >= '0' && c <= '9') { + d = c - '0'; + } else if (hex && c >= 'a' && c <= 'f') { + d = c - 'a' + 10; + } else if (hex && c >= 'A' && c <= 'F') { + d = c - 'A' + 10; + } else { + return -1; + } + value = value * (hex ? 16 : 10) + d; + if (value > 0x10FFFF) { + return -1; + } + } + return value; + } + + /// Runs of spaces become one, spaces around a line break go, and more + /// than one blank line in a row becomes one. + private static String tidy(StringBuilder raw) { + StringBuilder out = new StringBuilder(raw.length()); + int blank = 2; + int n = raw.length(); + int i = 0; + while (i < n) { + int eol = i; + while (eol < n && raw.charAt(eol) != '\n') { + eol++; + } + int from = i; + int to = eol; + while (from < to && raw.charAt(from) == ' ') { + from++; + } + while (to > from && raw.charAt(to - 1) == ' ') { + to--; + } + if (from == to) { + if (blank == 0) { + out.append('\n'); + } + blank++; + } else { + boolean space = false; + for (int p = from; p < to; p++) { + char c = raw.charAt(p); + if (c == ' ') { + space = true; + } else { + if (space) { + out.append(' '); + space = false; + } + out.append(c); + } + } + out.append('\n'); + blank = 0; + } + i = eol + 1; + } + int end = out.length(); + while (end > 0 && out.charAt(end - 1) == '\n') { + end--; + } + out.setLength(end); + if (end > 0) { + out.append('\n'); + } + return out.toString(); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/LogTransport.java b/vm/backend/src/com/codename1/backend/mail/LogTransport.java new file mode 100644 index 00000000000..3179a48490a --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/LogTransport.java @@ -0,0 +1,63 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.impl.backend.WireBuffer; + +/// Writes each mail to the log and sends nothing: `cn1.mail.url=log:`, for +/// reading a mail's link out of a terminal during development. +/// +/// It prints addresses and bodies, which no other part of the mail support +/// writes to a log. It is for a developer's own machine. +final class LogTransport extends MailTransport { + private static final int LIMIT = 8192; + + @Override + public String getName() { + return "log"; + } + + @Override + public void send(String from, String[] recipients, byte[] mime) { + StringBuilder sb = new StringBuilder(mime.length + 128); + sb.append("cn1: mail from ").append(from).append(" to"); + for (String r : recipients) { + sb.append(' ').append(r); + } + sb.append('\n'); + if (mime.length > LIMIT) { + byte[] head = new byte[LIMIT]; + System.arraycopy(mime, 0, head, 0, LIMIT); + sb.append(WireBuffer.fromUtf8(head)).append("\n... ").append(mime.length - LIMIT) + .append(" more bytes"); + } else { + sb.append(WireBuffer.fromUtf8(mime)); + } + System.out.println(sb.toString()); + } + + @Override + public void close() { + // Holds nothing. + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/Mail.java b/vm/backend/src/com/codename1/backend/mail/Mail.java new file mode 100644 index 00000000000..5c652952970 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/Mail.java @@ -0,0 +1,551 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.backend.mvc.Model; +import com.codename1.impl.backend.MailRuntime; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; + +/// One email: who it is for, what it says and what is attached. +/// +/// ```java +/// mail.send(Mail.to("ada@example.com") +/// .subject("Your order shipped") +/// .template("mail/shipped", model) +/// .attach("invoice.pdf", "application/pdf", pdf) +/// .build()); +/// ``` +/// +/// This one class is what Spring spreads over `SimpleMailMessage`, +/// `MimeMessage` and `MimeMessageHelper`. It is immutable; everything is +/// checked in [Builder#build], so a mail that exists can be sent: an address +/// that is not one, a header with a line break in it or an attachment with no +/// name fails there, with an IllegalArgumentException, never at the server. +/// +/// A mail with an HTML body and no text is given a text alternative derived +/// from the HTML, which is what keeps it out of spam folders and readable in +/// a client that shows text. The body is always UTF-8. +public final class Mail { + private static final Address[] NONE = new Address[0]; + private static final Part[] NO_PARTS = new Part[0]; + private static final String[] RESERVED = { + "From", "To", "Cc", "Bcc", "Reply-To", "Subject", "Date", "Message-ID", "MIME-Version", + "Content-Type", "Content-Transfer-Encoding", "Content-Disposition", "Content-ID", + "Sender", "Return-Path" + }; + + final Address from; + final Address replyTo; + final Address[] to; + final Address[] cc; + final Address[] bcc; + final String subject; + final String text; + final String html; + final String view; + final Model model; + final Part[] attachments; + final Part[] inlines; + /// Name, value, name, value. + final String[] headers; + + /// An attachment or an inline part. + static final class Part { + final String name; + final String type; + final byte[] data; + final String contentId; + + Part(String name, String type, byte[] data, String contentId) { + this.name = name; + this.type = type; + this.data = data; + this.contentId = contentId; + } + } + + private Mail(Address from, Address replyTo, Address[] to, Address[] cc, Address[] bcc, + String subject, String text, String html, String view, Model model, + Part[] attachments, Part[] inlines, String[] headers) { + this.from = from; + this.replyTo = replyTo; + this.to = to; + this.cc = cc; + this.bcc = bcc; + this.subject = subject; + this.text = text; + this.html = html; + this.view = view; + this.model = model; + this.attachments = attachments; + this.inlines = inlines; + this.headers = headers; + } + + /// Starts a mail to `addresses`, each `user@example.com` or + /// `Name `. + public static Builder to(String... addresses) { + return new Builder().to(addresses); + } + + /// Whether `address` is a bare `user@example.com` a mail can be sent to; + /// answers without throwing. + public static boolean isAddress(String address) { + return Address.isValid(address); + } + + /// Starts a mail with no recipient yet. + public static Builder builder() { + return new Builder(); + } + + /// Collects a mail's parts; [#build] checks them. + public static final class Builder { + private String from; + private String replyTo; + private final List to = new ArrayList(); + private final List cc = new ArrayList(); + private final List bcc = new ArrayList(); + private String subject; + private String text; + private String html; + private String view; + private Model model; + private final List attachments = new ArrayList(); + private final List inlines = new ArrayList(); + private final List headers = new ArrayList(); + + Builder() { + } + + /// The sender. Without one the mail is sent from `cn1.mail.from`. + public Builder from(String address) { + this.from = address; + return this; + } + + /// Where a reply goes when that is not the sender. + public Builder replyTo(String address) { + this.replyTo = address; + return this; + } + + /// Adds recipients. + public Builder to(String... addresses) { + add(to, addresses); + return this; + } + + /// Adds recipients every other recipient can see were copied. + public Builder cc(String... addresses) { + add(cc, addresses); + return this; + } + + /// Adds recipients no other recipient is told about. + public Builder bcc(String... addresses) { + add(bcc, addresses); + return this; + } + + private static void add(List list, String[] addresses) { + if (addresses != null) { + for (String a : addresses) { + list.add(a); + } + } + } + + /// The subject line; any language, one line. + public Builder subject(String subject) { + this.subject = subject; + return this; + } + + /// The plain text body. + public Builder text(String text) { + this.text = text; + return this; + } + + /// The HTML body. + public Builder html(String html) { + this.html = html; + this.view = null; + this.model = null; + return this; + } + + /// The HTML body, rendered from the template + /// `templates/.html` with `model` when the mail is sent. The + /// text body is derived from what was rendered, unless [#text] gave + /// one. + public Builder template(String view, Model model) { + this.view = view; + this.model = model == null ? new Model() : model; + this.html = null; + return this; + } + + /// Attaches a file. + /// + /// - `name`: the file name the reader sees + /// - `type`: its media type, such as `application/pdf`; null for + /// `application/octet-stream` + /// - `data`: its content, which the mail keeps and does not copy + public Builder attach(String name, String type, byte[] data) { + attachments.add(new Part(name, type, data, null)); + return this; + } + + /// Embeds an image the HTML body shows with ``. + public Builder inline(String contentId, String type, byte[] data) { + inlines.add(new Part(null, type, data, contentId)); + return this; + } + + /// Adds a header of the application's own, such as `List-Unsubscribe` + /// or `X-Campaign`. The headers a mail is built from -- the addresses, + /// the subject, the date, the content headers -- cannot be set here. + public Builder header(String name, String value) { + headers.add(name); + headers.add(value); + return this; + } + + /// Checks everything and answers the mail, or throws + /// IllegalArgumentException naming what is wrong. + public Mail build() { + Address[] toList = parse(to); + Address[] ccList = parse(cc); + Address[] bccList = parse(bcc); + if (toList.length + ccList.length + bccList.length == 0) { + throw new IllegalArgumentException("A mail needs at least one recipient"); + } + if (subject != null) { + checkLine("The subject", subject); + } + if (text == null && html == null && view == null) { + throw new IllegalArgumentException("A mail needs a body: text(), html() or " + + "template()"); + } + if (!inlines.isEmpty() && html == null && view == null) { + throw new IllegalArgumentException("An inline part needs an HTML body to " + + "show it"); + } + Part[] files = new Part[attachments.size()]; + for (int i = 0; i < files.length; i++) { + Part p = attachments.get(i); + if (p.name == null || p.name.trim().length() == 0) { + throw new IllegalArgumentException("An attachment needs a file name"); + } + if (p.name.length() > 255) { + throw new IllegalArgumentException("An attachment's file name is longer " + + "than 255 characters"); + } + int encoded = 0; + for (int c = 0; c < p.name.length(); c++) { + char ch = p.name.charAt(c); + boolean plain = ch >= 'a' && ch <= 'z' || ch >= 'A' && ch <= 'Z' + || ch >= '0' && ch <= '9' || ch == '.' || ch == '-' || ch == '_'; + encoded += plain ? 1 : ch < 0x80 ? 3 : ch < 0x800 ? 6 : 9; + } + if (encoded > 900) { + // A name with anything outside ASCII is written three + // bytes a byte, but for letters, digits and ._-, on one + // line, and a line may be 998. + throw new IllegalArgumentException("An attachment's file name is too " + + "long once encoded; shorten it to about a hundred characters"); + } + checkLine("An attachment's file name", p.name); + files[i] = new Part(p.name, checkType(p.type), checkData(p.data), null); + } + Part[] embedded = new Part[inlines.size()]; + for (int i = 0; i < embedded.length; i++) { + Part p = inlines.get(i); + if (p.contentId == null || p.contentId.length() == 0) { + throw new IllegalArgumentException("An inline part needs a content id"); + } + for (int c = 0; c < p.contentId.length(); c++) { + char ch = p.contentId.charAt(c); + if (ch <= ' ' || ch > 126 || ch == '<' || ch == '>') { + throw new IllegalArgumentException("An inline part's content id may " + + "hold only visible ASCII characters other than < and >"); + } + } + embedded[i] = new Part(null, checkType(p.type), checkData(p.data), p.contentId); + } + String[] extra = new String[headers.size()]; + for (int i = 0; i < extra.length; i += 2) { + extra[i] = checkHeaderName(headers.get(i)); + String value = headers.get(i + 1); + if (value == null) { + throw new IllegalArgumentException("The header " + extra[i] + " has no value"); + } + checkLine("The header " + extra[i], value); + if (MimeWriter.longWord(value)) { + // A line may be 998 bytes and folds only at a space. + throw new IllegalArgumentException("The header " + extra[i] + " has a run " + + "of more than 900 characters without a space, which no mail " + + "server takes"); + } + extra[i + 1] = value; + } + String plain = text; + if (plain == null && html != null) { + plain = HtmlText.strip(html); + } + return new Mail(from == null ? null : Address.parse(from), + replyTo == null ? null : Address.parse(replyTo), toList, ccList, bccList, + subject, plain, html, view, model, + files.length == 0 ? NO_PARTS : files, + embedded.length == 0 ? NO_PARTS : embedded, extra); + } + + private static Address[] parse(List list) { + if (list.isEmpty()) { + return NONE; + } + Address[] out = new Address[list.size()]; + for (int i = 0; i < out.length; i++) { + out[i] = Address.parse(list.get(i)); + } + return out; + } + + /// A header value is one line. A line break in one is how a mail + /// header is injected -- "subject\r\nBcc: someone" -- so it is refused + /// rather than folded or stripped. + private static void checkLine(String what, String value) { + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '\r' || c == '\n' || c == 0) { + throw new IllegalArgumentException(what + " has a line break or a NUL in " + + "it; a header is one line"); + } + } + } + + private static String checkHeaderName(String name) { + if (name == null || name.length() == 0) { + throw new IllegalArgumentException("A header needs a name"); + } + for (int i = 0; i < name.length(); i++) { + char c = name.charAt(i); + if (c <= ' ' || c > 126 || c == ':') { + throw new IllegalArgumentException("A header name may hold only visible " + + "ASCII characters other than the colon"); + } + } + if (name.length() > 76) { + // A value folds; a name is on one line with what follows it. + throw new IllegalArgumentException("A header name may be 76 characters"); + } + for (String r : RESERVED) { + if (r.equalsIgnoreCase(name)) { + throw new IllegalArgumentException("The header " + name + " is written " + + "from the mail itself and cannot be set with header()"); + } + } + return name; + } + + private static String checkType(String type) { + if (type == null) { + return "application/octet-stream"; + } + int slash = type.indexOf('/'); + boolean ok = slash > 0 && slash < type.length() - 1; + for (int i = 0; ok && i < type.length(); i++) { + char c = type.charAt(i); + ok = c > ' ' && c <= 126 && c != ';' && c != '"' && c != '\\' && c != ',' + && c != '(' && c != ')' && c != '<' && c != '>' && c != '@' + && (c != '/' || i == slash); + } + if (!ok) { + throw new IllegalArgumentException("\"" + type + "\" is not a media type; " + + "one looks like application/pdf"); + } + return type; + } + + private static byte[] checkData(byte[] data) { + if (data == null) { + throw new IllegalArgumentException("An attachment has no content"); + } + return data; + } + } + + /// This mail with its template rendered and its sender filled in: what is + /// encoded, sent and kept by a test's capture. + Mail resolved(Address defaultFrom) throws IOException { + Address sender = from != null ? from : defaultFrom; + if (sender == null) { + throw new MailException("The mail has no sender: call from(), or set " + + "cn1.mail.from in application.properties"); + } + if (view == null && sender == from) { //NOPMD CompareObjectsWithEquals - the same object means nothing to fill in + return this; + } + String body = html; + String plain = text; + if (view != null) { + MailRuntime.TemplateRenderer renderer = MailRuntime.renderer(); + if (renderer == null) { + throw new MailException("The mail names the template \"" + view + "\" and " + + "this application was built with no mail templates: put it in " + + "src/main/resources/templates/" + view + ".html"); + } + try { + body = WireBuffer.fromUtf8(renderer.render(view, model)); + } catch (IllegalArgumentException err) { + throw new MailException("The mail names the template \"" + view + "\", which " + + "this application does not have: src/main/resources/templates/" + view + + ".html", err); + } + if (plain == null) { + plain = HtmlText.strip(body); + } + } + return new Mail(sender, replyTo, to, cc, bcc, subject, plain, body, null, null, + attachments, inlines, headers); + } + + /// The addresses the server is told to deliver to: everyone in to, cc and + /// bcc, once each. + List recipients() { + List out = new ArrayList(to.length + cc.length + bcc.length); + collect(out, to); + collect(out, cc); + collect(out, bcc); + return out; + } + + private static void collect(List out, Address[] list) { + for (Address a : list) { + if (!out.contains(a.spec)) { + out.add(a.spec); + } + } + } + + /// Whether every address is ASCII; a mail with one that is not can only + /// go through a server that offers SMTPUTF8. + boolean isAsciiEnvelope() { + return (from == null || from.ascii) && ascii(to) && ascii(cc) && ascii(bcc); + } + + private static boolean ascii(Address[] list) { + for (Address a : list) { + if (!a.ascii) { + return false; + } + } + return true; + } + + private static List specs(Address[] list) { + List out = new ArrayList(list.length); + for (Address a : list) { + out.add(a.spec); + } + return Collections.unmodifiableList(out); + } + + /// The sender as it was given, or null when the mail takes the default. + public String getFrom() { + return from == null ? null : from.toString(); + } + + /// The reply address, or null. + public String getReplyTo() { + return replyTo == null ? null : replyTo.toString(); + } + + /// The addresses of the recipients, without their display names. + public List getTo() { + return specs(to); + } + + /// The addresses copied. + public List getCc() { + return specs(cc); + } + + /// The addresses copied without the other recipients being told. + public List getBcc() { + return specs(bcc); + } + + /// The subject, or null. + public String getSubject() { + return subject; + } + + /// The text body. For a mail built from a template this is null until the + /// mail is sent; the mail a test captures has it. + public String getText() { + return text; + } + + /// The HTML body, or null. For a mail built from a template this is null + /// until the mail is sent; the mail a test captures has it. + public String getHtml() { + return html; + } + + /// The file names of the attachments, in order. + public List getAttachmentNames() { + List out = new ArrayList(attachments.length); + for (Part p : attachments) { + out.add(p.name); + } + return Collections.unmodifiableList(out); + } + + /// The content of the attachment called `name`, or null. + public byte[] getAttachment(String name) { + for (Part p : attachments) { + if (p.name.equals(name)) { + return p.data; + } + } + return null; + } + + /// The value of a header added with [Builder#header], or null. The name + /// is matched without regard to case. + public String getHeader(String name) { + for (int i = 0; i < headers.length; i += 2) { + if (headers[i].equalsIgnoreCase(name)) { + return headers[i + 1]; + } + } + return null; + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailAuthenticationException.java b/vm/backend/src/com/codename1/backend/mail/MailAuthenticationException.java new file mode 100644 index 00000000000..a2f26b138ad --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailAuthenticationException.java @@ -0,0 +1,36 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +/// The mail server refused the credentials it was given, or offers no way to +/// present them that is safe to use. +/// +/// Not permanent: the mail is as good as it was and the configuration is what +/// is wrong, so a queued mail is tried again -- a rotated password must not +/// cost every mail sent before it was corrected. +public class MailAuthenticationException extends MailSendException { + /// - `code`: the server's reply code, or 0 + public MailAuthenticationException(String message, int code) { + super(message, code, false); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailException.java b/vm/backend/src/com/codename1/backend/mail/MailException.java new file mode 100644 index 00000000000..b12a4ba613f --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailException.java @@ -0,0 +1,46 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import java.io.IOException; + +/// A mail that could not be sent: the server could not be reached, refused +/// the sender, or refused the message. +public class MailException extends IOException { + /// A failure with a description of what was being done. + public MailException(String message) { + super(message); + } + + /// A failure caused by another. + public MailException(String message, Throwable cause) { + super(message, cause); + } + + /// Whether sending the same mail again can never succeed: the server + /// answered with a permanent refusal. A failure that is not permanent -- + /// a timeout, a dropped connection, a "try again later" -- is retried. + public boolean isPermanent() { + return false; + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailMetrics.java b/vm/backend/src/com/codename1/backend/mail/MailMetrics.java new file mode 100644 index 00000000000..b2cf679b59a --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailMetrics.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.backend.Span; +import com.codename1.backend.Tracing; +import com.codename1.backend.metrics.Counter; +import com.codename1.backend.metrics.Gauge; +import com.codename1.backend.metrics.Metrics; +import com.codename1.impl.backend.MailRuntime; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicBoolean; + +/// The mail counters. Registered the first time a mail is settled, so an +/// application that sends none reports none. +final class MailMetrics { + private static final class Holder { + static final Counter SENT = Metrics.counter("cn1.mail.sent", + "Mails a transport accepted", "{mail}"); + static final Counter FAILED = Metrics.counter("cn1.mail.failed", + "Mails given up on: refused, or out of attempts", "{mail}"); + static final Counter REFUSED = Metrics.counter("cn1.mail.refused", + "Mails given up on because the server refused them for good", "{mail}"); + static final Counter RETRIED = Metrics.counter("cn1.mail.retried", + "Attempts that failed and will be made again", "{attempt}"); + } + + private static final AtomicBoolean OUTBOX = new AtomicBoolean(); + + private MailMetrics() { + } + + /// Registers the outbox gauge, once for the process. It reads whichever + /// mail support is open, so a server restarted in the same process -- a + /// test -- reports its own. + static void outbox() { + if (!OUTBOX.compareAndSet(false, true)) { + return; + } + Metrics.gauge("cn1.mail.outbox.pending", + "Mails staged and not yet handed to the transport", "{mail}", + new Gauge.Source() { + @Override + public double read() { + MailRuntime.Probe probe = MailRuntime.probe(); + return probe == null ? 0 : probe.pending(); + } + }); + } + + /// One hand-over to the transport, under a span when a tracer is + /// installed. The span says which transport and how many recipients, and + /// never an address: a trace is read by more people than a mailbox. + static void transmit(MailTransport transport, String from, String[] recipients, byte[] mime) + throws IOException { + Span span = Tracing.startSpan("mail send"); + try { + if (span.isRecording()) { + span.setAttribute("mail.transport", transport.getName()); + span.setAttribute("mail.recipients", (long) recipients.length); + span.setAttribute("mail.size", (long) mime.length); + } + transport.send(from, recipients, mime); + } catch (IOException err) { + span.recordException(err); + throw err; + } catch (RuntimeException err) { + span.recordException(err); + throw err; + } finally { + span.end(); + } + } + + static void sent() { + Holder.SENT.increment(); + } + + static void failed(boolean refused) { + Holder.FAILED.increment(); + if (refused) { + Holder.REFUSED.increment(); + } + } + + static void retried() { + Holder.RETRIED.increment(); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailQueue.java b/vm/backend/src/com/codename1/backend/mail/MailQueue.java new file mode 100644 index 00000000000..1cca6e0f6bf --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailQueue.java @@ -0,0 +1,614 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.backend.Base64; +import com.codename1.backend.DataSource; +import com.codename1.backend.Database; +import com.codename1.backend.Transactions; +import com.codename1.backend.messaging.RetryPolicy; +import com.codename1.backend.sql.Dialect; +import com.codename1.impl.backend.BackendAccess; +import java.io.IOException; +import java.util.ArrayList; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; + +/// The queue between a call to send and the transport: one thread that hands +/// mails over, tries again when that fails, and gives up on a mail the server +/// will never take. +/// +/// With a data source the queue is the table `cn1_mail_outbox`, written in +/// the caller's transaction -- the transactional outbox the messaging support +/// uses, for the same reason. A rollback sends nothing. A commit never loses +/// the mail: a row whose send did not happen, because the process died or the +/// server was down, is sent by the sweep every server runs. A mail can +/// therefore be sent twice and never zero times. +/// +/// A mail that fails for good -- the server refused it, or it used up its +/// attempts -- moves to `cn1_mail_dead` with the reason, where an operator +/// can read it; nothing sends from that table. +/// +/// Without a data source the queue is memory: retried the same way, and gone +/// when the process stops. +/// +/// Sending is on this thread and never on the caller's for a second reason +/// than the transaction: an SMTP exchange is several round trips, and a +/// request handler that waited for one would hold its thread for all of them. +final class MailQueue { + static final String TABLE = "cn1_mail_outbox"; + static final String DEAD = "cn1_mail_dead"; + /// How long a row a sweep claimed is left alone by the other sweeps. + /// Longer than a send can take -- a connection, a TLS handshake, a sign-in + /// and a large body each wait up to the read timeout -- since a claim + /// that ran out mid-send would let another server send the same mail. + private static final long CLAIM = 600000; + + private final MailTransport transport; + private final DataSource db; + private final RetryPolicy retry; + private final long sweepAfter; + private final AtomicBoolean running = new AtomicBoolean(true); + /// Set when [#close] has stopped waiting: the transport closes next, so + /// what the thread still holds is left to the next start. + private final AtomicBoolean abandoned = new AtomicBoolean(); + private final AtomicInteger depth = new AtomicInteger(); + private final Object pause = new Object(); + /// Mails waiting for the thread, each with the time it is due. + private final List waiting = new ArrayList(); + /// The rows this process holds in [#waiting], which its sweep leaves alone. + private final Set held = new HashSet(); + private boolean created; + private Thread thread; + + private static final class Item { + Object id; + final String from; + final String[] recipients; + final byte[] mime; + int attempt; + long due; + /// What the row's `next_attempt` holds, which is what a claim of it + /// is conditional on; 0 once this process has claimed it. + long stamp; + + Item(String from, String[] recipients, byte[] mime) { + this.from = from; + this.recipients = recipients; + this.mime = mime; + } + } + + MailQueue(MailTransport transport, DataSource db, RetryPolicy retry, long sweepAfterMillis) { + this.transport = transport; + this.db = db; + this.retry = retry; + this.sweepAfter = Math.max(100, sweepAfterMillis); + if (db != null) { + // The tables are made here, not by the first send. MySQL and + // MariaDB end the open transaction on any DDL, a CREATE TABLE IF + // NOT EXISTS of a table that exists included: a first send inside + // a transaction then wrote its row outside it, and a mail the + // caller rolled back was sent by the sweep. A database that is + // down now is tried again by the sweep, and by a send as a last + // resort. + try { + ensureTables(); + } catch (IOException | RuntimeException err) { + System.err.println("cn1: the mail outbox tables could not be created yet: " + + err.getMessage()); + } + // Not on the first send: what a stopped server left in the table + // is sent by the next one whether or not it has mail of its own. + start(); + } + } + + /// Mails accepted and not yet handed over or given up on, in this process. + int depth() { + return depth.get(); + } + + private void ensureTables() throws IOException { + synchronized (this) { + if (created) { + return; + } + } + Dialect d = db.dialect(); + String text = d.columnType(Dialect.TEXT); + String big = d.columnType(Dialect.BIGINT); + String number = d.columnType(Dialect.INTEGER); + db.execute("CREATE TABLE IF NOT EXISTS " + TABLE + " (id " + + d.generatedKeyColumn(Dialect.BIGINT) + + ", mail_from VARCHAR(320) NOT NULL, recipients " + text + " NOT NULL, mime " + + text + " NOT NULL, attempt " + number + " NOT NULL, next_attempt " + big + + " NOT NULL, last_error VARCHAR(500))", new Object[0]); + db.execute("CREATE TABLE IF NOT EXISTS " + DEAD + " (id " + + d.generatedKeyColumn(Dialect.BIGINT) + + ", mail_from VARCHAR(320) NOT NULL, recipients " + text + " NOT NULL, mime " + + text + " NOT NULL, attempts " + number + " NOT NULL, error VARCHAR(500), died_at " + + big + " NOT NULL)", new Object[0]); + synchronized (this) { + created = true; + } + } + + /// Takes a mail for delivery: now when no transaction is open, and when + /// it commits otherwise. + void submit(String from, String[] recipients, byte[] mime) throws IOException { + if (db == null && !running.get()) { + // Nothing would ever send it, and nothing keeps it. + throw new MailException("The mail support is closed"); + } + start(); + final Item item = new Item(from, recipients, mime); + if (db != null) { + ensureTables(); + // Due after a sweep age, so the row is the fallback and the + // commit callback below is what normally sends it. + item.stamp = System.currentTimeMillis() + sweepAfter; + item.id = Long.valueOf(db.insert("INSERT INTO " + TABLE + " (mail_from, recipients," + + " mime, attempt, next_attempt) VALUES (?, ?, ?, 0, ?)", + new Object[]{from, join(recipients), Base64.encode(mime), + Long.valueOf(item.stamp)}, "id")); + } + if (Transactions.isActive()) { + Transactions.registerSynchronization(new Transactions.Synchronization() { + @Override + public void afterCommit() { + enqueue(item); + } + }); + } else { + enqueue(item); + } + } + + private void enqueue(Item item) { + if (db == null && !running.get()) { + // A transaction that committed after the server stopped. + System.err.println("cn1: a mail to " + item.recipients.length + " recipient(s) is " + + "lost: the mail support closed before its transaction committed"); + return; + } + synchronized (pause) { + waiting.add(item); + if (item.id != null) { + held.add(item.id); + } + depth.incrementAndGet(); + pause.notifyAll(); + } + } + + private static String join(String[] recipients) { + StringBuilder sb = new StringBuilder(); + for (String r : recipients) { + if (sb.length() > 0) { + sb.append('\n'); + } + sb.append(r); + } + return sb.toString(); + } + + private static String[] split(String recipients) { + List out = new ArrayList(); + int from = 0; + while (from <= recipients.length()) { + int end = recipients.indexOf('\n', from); + if (end < 0) { + end = recipients.length(); + } + if (end > from) { + out.add(recipients.substring(from, end)); + } + from = end + 1; + } + return out.toArray(new String[out.size()]); + } + + /// A failure as it may be logged: the server's words with every address + /// in them removed. A mail server quotes the recipient back in its reply, + /// and a log is not where a user's address belongs. + static String redact(String message) { + if (message == null) { + return "no reason given"; + } + if (message.indexOf('@') < 0) { + return message; + } + StringBuilder sb = new StringBuilder(message.length()); + int from = 0; + while (from <= message.length()) { + int end = message.indexOf(' ', from); + if (end < 0) { + end = message.length(); + } + String word = message.substring(from, end); + if (sb.length() > 0) { + sb.append(' '); + } + sb.append(word.indexOf('@') >= 0 ? "
" : word); + from = end + 1; + } + return sb.toString(); + } + + private static String reason(Throwable err) { + String m = err.getMessage(); + if (m == null) { + m = err.toString(); + } + return m.length() > 500 ? m.substring(0, 500) : m; + } + + /// Makes a claimed row due now, for the next start to send. + private void giveBack(Item item) { + try { + db.execute("UPDATE " + TABLE + " SET next_attempt = ? WHERE id = ?", + new Object[]{Long.valueOf(System.currentTimeMillis()), item.id}); + } catch (IOException | RuntimeException err) { + // Its claim runs out by itself. + System.err.println("cn1: a mail waits in " + TABLE + " for its claim to run " + + "out: " + err.getMessage()); + } + } + + private void deliver(Item item) { + if (item.id != null && abandoned.get()) { + // Its row is there for the next start. One the sweep claimed is + // given back, or that start would wait out the claim. + if (item.stamp == 0) { + giveBack(item); + } + settle(item); + return; + } + if (item.id != null && item.stamp != 0 && !claim(item)) { + // Another server's. + settle(item); + return; + } + Throwable failure = null; + try { + MailMetrics.transmit(transport, item.from, item.recipients, item.mime); + } catch (IOException err) { + failure = err; + } catch (RuntimeException err) { + failure = err; + } + if (failure == null) { + MailMetrics.sent(); + settle(item); + if (item.id != null) { + try { + db.execute("DELETE FROM " + TABLE + " WHERE id = ?", new Object[]{item.id}); + } catch (IOException | RuntimeException err) { + System.err.println("cn1: a sent mail stays in " + TABLE + " and will be " + + "sent again: " + err.getMessage()); + } + } + return; + } + boolean permanent = failure instanceof MailException + && ((MailException) failure).isPermanent(); + if (item.id != null && abandoned.get() && !permanent) { + // The transport closed under it. That is no attempt at sending: + // the row is given back as it was, due now, for the next start. + // A refusal that is final was the server's answer and is buried + // below; sending that mail again could reach a recipient twice. + giveBack(item); + settle(item); + return; + } + item.attempt++; + if (permanent || item.attempt >= retry.getMaxAttempts()) { + MailMetrics.failed(permanent); + settle(item); + bury(item, failure, permanent); + return; + } + MailMetrics.retried(); + long delay = retry.delayAfter(item.attempt); + item.due = System.currentTimeMillis() + delay; + System.err.println("cn1: a mail to " + item.recipients.length + " recipient(s) was not " + + "sent and will be tried again in " + delay + "ms (attempt " + item.attempt + + " of " + retry.getMaxAttempts() + "): " + redact(reason(failure))); + if (item.id != null) { + try { + db.execute("UPDATE " + TABLE + " SET attempt = ?, next_attempt = ?, last_error = ?" + + " WHERE id = ?", new Object[]{Integer.valueOf(item.attempt), + Long.valueOf(item.due + sweepAfter), reason(failure), item.id}); + item.stamp = item.due + sweepAfter; + } catch (IOException | RuntimeException err) { + // The row keeps this process's claim; the sweep that finds + // it once the claim runs out carries on from there. + System.err.println("cn1: " + TABLE + " could not record a failed attempt: " + + err.getMessage()); + settle(item); + return; + } + } + synchronized (pause) { + // Still held, still counted: only its due time changed. + waiting.add(item); + } + } + + /// Makes the row of `item` this process's to send, by moving its time + /// forward from the one it was last given. Every server on the database + /// sweeps the table, and the set of rows a process holds is its own: a + /// row this process is slow to reach -- a backlog, a slow server -- comes + /// due and is another sweep's to take. Whoever moves the time has it; + /// false is a row somebody else moved, or one that is gone. + private boolean claim(Item item) { + try { + long until = System.currentTimeMillis() + CLAIM; + if (db.execute("UPDATE " + TABLE + " SET next_attempt = ? WHERE id = ? AND " + + "next_attempt = ?", new Object[]{Long.valueOf(until), item.id, + Long.valueOf(item.stamp)}) != 1) { + return false; + } + item.stamp = 0; + return true; + } catch (IOException | RuntimeException err) { + // The row is there for the sweep, which sends it when the + // database answers again. + System.err.println("cn1: a mail waits in " + TABLE + " for the next sweep: " + + err.getMessage()); + return false; + } + } + + private void settle(Item item) { + depth.decrementAndGet(); + if (item.id != null) { + synchronized (pause) { + held.remove(item.id); + } + } + } + + /// A row moved to the dead table, as the body of a transaction. + private static final class Bury implements DataSource.Work { + private final Object[] dead; + private final Object id; + + Bury(Object[] dead, Object id) { + this.dead = dead; + this.id = id; + } + + @Override + public Object run(Database tx) throws Exception { + tx.execute("INSERT INTO " + DEAD + " (mail_from, recipients, mime, attempts," + + " error, died_at) VALUES (?, ?, ?, ?, ?, ?)", dead); + tx.execute("DELETE FROM " + TABLE + " WHERE id = ?", new Object[]{id}); + return null; + } + } + + private void bury(Item item, Throwable failure, boolean permanent) { + System.err.println("cn1: a mail to " + item.recipients.length + " recipient(s) was " + + (permanent ? "refused" : "given up on after " + item.attempt + " attempts") + + (item.id != null ? " and moved to " + DEAD : " and is lost") + ": " + + redact(reason(failure))); + if (item.id == null) { + return; + } + final Object[] dead = {item.from, join(item.recipients), Base64.encode(item.mime), + Integer.valueOf(item.attempt), reason(failure), + Long.valueOf(System.currentTimeMillis())}; + final Object id = item.id; + try { + // Both or neither: a row copied and not removed would be copied + // again by every sweep. + db.inTransaction(new Bury(dead, id)); + } catch (Exception err) { + System.err.println("cn1: a failed mail could not be moved to " + DEAD + ": " + + err.getMessage()); + // Marked as out of attempts where it is, so that the sweep that + // finds it moves it and does not send it: a mail the server + // refused may have reached some of its recipients already. + try { + db.execute("UPDATE " + TABLE + " SET attempt = ?, next_attempt = ?, last_error = ?" + + " WHERE id = ?", new Object[]{ + Integer.valueOf(Math.max(item.attempt, retry.getMaxAttempts())), + Long.valueOf(System.currentTimeMillis() + sweepAfter), + reason(failure), id}); + } catch (IOException | RuntimeException again) { + // The database is not answering. The row keeps its claim, and + // is tried again when that runs out: sent twice, never lost. + System.err.println("cn1: nor could it be marked as failed: " + + again.getMessage()); + } + } + } + + /// Sends every row that is due and that this process is not already + /// holding; answers how many it tried. + int sweep() throws IOException { + ensureTables(); + long now = System.currentTimeMillis(); + List rows = db.query("SELECT id, mail_from, recipients, mime, attempt, next_attempt, last_error FROM " + + TABLE + " WHERE next_attempt <= ? ORDER BY id " + db.dialect().limit(50, 0), + new Object[]{Long.valueOf(now)}); + int tried = 0; + for (Object o : rows) { + Map row = (Map) o; + Object id = number(row.get("id")); + synchronized (pause) { + if (held.contains(id)) { + continue; + } + } + Object mime = row.get("mime"); + if (!(mime instanceof String)) { + // Never an empty mail in its place, counted as sent. + System.err.println("cn1: a row of " + TABLE + " has no mail that can be read " + + "and is left where it is"); + continue; + } + // Claimed by moving its time forward, so the sweep of another + // process leaves it alone while this one sends it. + if (db.execute("UPDATE " + TABLE + " SET next_attempt = ? WHERE id = ? AND " + + "next_attempt = ?", new Object[]{ + // From now, not from when the sweep began: the rows + // before this one may each have taken a minute. + Long.valueOf(System.currentTimeMillis() + CLAIM), id, + row.get("next_attempt")}) != 1) { + continue; + } + Item item = new Item(String.valueOf(row.get("mail_from")), + split(String.valueOf(row.get("recipients"))), + Base64.decode((String) mime)); + item.id = id; + Object attempt = row.get("attempt"); + item.attempt = attempt instanceof Number ? ((Number) attempt).intValue() : 0; + synchronized (pause) { + held.add(id); + } + depth.incrementAndGet(); + if (item.attempt >= retry.getMaxAttempts()) { + // Out of attempts already, and still here because the move + // to the dead table failed: move it, do not send it again. + // Counted as failed when it failed. + settle(item); + bury(item, new MailException(String.valueOf(row.get("last_error"))), false); + } else { + deliver(item); + } + tried++; + } + return tried; + } + + /// A key as the set of held rows keeps it: an id read back from a query + /// is not always the type the insert answered. + private static Object number(Object id) { + return id instanceof Number ? (Object) Long.valueOf(((Number) id).longValue()) : id; + } + + private void start() { + synchronized (this) { + if (thread != null || !running.get()) { + return; + } + thread = new Thread(BackendAccess.get().inServer(new Runnable() { + @Override + public void run() { + loop(); + } + }), "cn1-mail"); + thread.setDaemon(true); + thread.start(); + } + } + + private void loop() { + long nextSweep = 0; + while (true) { + long now = System.currentTimeMillis(); + List due = new ArrayList(); + synchronized (pause) { + for (int i = waiting.size() - 1; i >= 0; i--) { + if (waiting.get(i).due <= now) { + due.add(0, waiting.remove(i)); + } + } + } + for (Item item : due) { + deliver(item); + } + if (db != null && running.get() && System.currentTimeMillis() >= nextSweep) { + try { + sweep(); + } catch (IOException | RuntimeException err) { + // A row it cannot read must not end the thread every + // committed mail waits on. + System.err.println("cn1: the mail sweep failed: " + err.getMessage()); + } + nextSweep = System.currentTimeMillis() + sweepAfter; + } + synchronized (pause) { + long wake = db != null ? nextSweep : Long.MAX_VALUE; + boolean ready = false; + now = System.currentTimeMillis(); + for (Item item : waiting) { + if (item.due <= now) { + ready = true; + } + wake = Math.min(wake, item.due); + } + if (!running.get() && !ready) { + // What is left is waiting out a retry delay. With a data + // source the row is still there for the next start. + if (db == null && !waiting.isEmpty()) { + System.err.println("cn1: " + waiting.size() + " mail(s) waiting to be " + + "tried again are lost: the server stopped and has no data " + + "source to keep them in"); + } + return; + } + if (!ready) { + try { + long left = wake - now; + if (left > 0) { + pause.wait(left == Long.MAX_VALUE - now ? 0 : left); + } + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + return; + } + } + } + } + } + + /// Stops the thread once it has handed over what is due, waiting at most + /// `waitMillis` for that. + void close(long waitMillis) { + running.set(false); + Thread t; + synchronized (this) { + t = thread; + } + synchronized (pause) { + pause.notifyAll(); + } + if (t != null && t != Thread.currentThread()) { //NOPMD CompareObjectsWithEquals - thread identity + try { + t.join(waitMillis); + } catch (InterruptedException err) { + Thread.currentThread().interrupt(); + } + } + abandoned.set(true); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailSendException.java b/vm/backend/src/com/codename1/backend/mail/MailSendException.java new file mode 100644 index 00000000000..b43a126838e --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailSendException.java @@ -0,0 +1,50 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +/// A mail the server refused, with the reply it gave. +public class MailSendException extends MailException { + private final int code; + private final boolean permanent; + + /// - `code`: the three-digit SMTP reply code, an HTTP status from a + /// provider's API, or 0 when there was none + /// - `permanent`: whether the server will give the same answer next time. + /// The transport decides: an SMTP 4xx means "later" and an HTTP 4xx + /// mostly means "never", so the number alone does not say. + public MailSendException(String message, int code, boolean permanent) { + super(message); + this.code = code; + this.permanent = permanent; + } + + /// The server's reply code, or 0. + public int getCode() { + return code; + } + + @Override + public boolean isPermanent() { + return permanent; + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailSender.java b/vm/backend/src/com/codename1/backend/mail/MailSender.java new file mode 100644 index 00000000000..96b8af18589 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailSender.java @@ -0,0 +1,93 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import java.io.IOException; +import java.util.List; + +/// Sends mail through the server's transport: what an application injects, +/// as it would Spring's `JavaMailSender`. +/// +/// ```java +/// @Service +/// public class Signup { +/// private final MailSender mail; +/// public Signup(MailSender mail) { this.mail = mail; } +/// +/// @Transactional +/// public void register(Account account) throws IOException { +/// accounts.save(account); +/// mail.send(Mail.to(account.getEmail()) +/// .subject("Welcome") +/// .template("mail/welcome", new Model().addAttribute("name", account.getName())) +/// .build()); +/// } +/// } +/// ``` +/// +/// [#send] returns at once. The mail is encoded on the calling thread and +/// handed to the server's mail thread, which delivers it, tries again when +/// the mail server is unreachable and gives up when it refuses. Inside a +/// database transaction the mail is written to an outbox table in that +/// transaction and sent after the commit, so a rollback sends nothing and a +/// crash between the commit and the send sends it when the server is back. +/// +/// Where it goes is `cn1.mail.url`; see [Mailers]. Without one, mail is kept +/// in memory and [#sent] answers it, which is how a test reads what its code +/// sent. +public final class MailSender { + /// Takes `mail` for delivery and returns without waiting for it. + public void send(Mail mail) throws IOException { + Mailers.current().send(mail); + } + + /// Takes several mails; they leave over one connection where the + /// transport has connections. + public void send(Mail... mails) throws IOException { + Mailers.current().send(mails); + } + + /// Sends `mail` on the calling thread and returns when the mail server + /// has taken it, or throws a [MailException] saying why it did not. No + /// queue, no retry, and no regard for a transaction. + public void sendNow(Mail mail) throws IOException { + Mailers.current().sendNow(mail); + } + + /// The mails sent while the server has no transport, oldest first, each + /// with its template rendered and its sender filled in. + public List sent() throws IOException { + return Mailers.current().sent(); + } + + /// Forgets what [#sent] answers; a test calls it before it acts. + public void clearSent() throws IOException { + Mailers.current().clearSent(); + } + + /// The sender of the calling thread's server, for code no bean is + /// injected into. + public static MailSender current() { + return new MailSender(); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MailTransport.java b/vm/backend/src/com/codename1/backend/mail/MailTransport.java new file mode 100644 index 00000000000..cf487e7d38b --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MailTransport.java @@ -0,0 +1,58 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import java.io.IOException; + +/// One way of handing an encoded mail to something that will deliver it: an +/// SMTP server, a provider's API. +/// +/// A transport is given the envelope and the bytes and nothing else. It does +/// not queue, retry or keep anything; the server's [Mailer] does that around +/// it, the same way for every transport. It is called from the mail thread +/// and from any thread that calls [MailSender#sendNow], so it is safe to call +/// from several at once. +public abstract class MailTransport { + /// What this transport is: `smtp`, `ses`, `memory`, `log`. + public abstract String getName(); + + /// Hands one mail over, returning when the other side has taken it. + /// + /// Throws a [MailException] whose [MailException#isPermanent] says + /// whether trying again could help; any other IOException is taken to be + /// a failure of the connection, and is retried. + /// + /// - `from`: the address a bounce goes to, without a display name + /// - `recipients`: the addresses to deliver to, without display names + /// - `mime`: the whole message, CRLF line endings, not dot-stuffed + public abstract void send(String from, String[] recipients, byte[] mime) throws IOException; + + /// Whether the transport could reach its server the last time it tried. + /// One that has not tried yet is healthy. + public boolean isHealthy() { + return true; + } + + /// Closes connections it holds. Never throws. + public abstract void close(); +} diff --git a/vm/backend/src/com/codename1/backend/mail/Mailer.java b/vm/backend/src/com/codename1/backend/mail/Mailer.java new file mode 100644 index 00000000000..938fe261499 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/Mailer.java @@ -0,0 +1,207 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.backend.Crypto; +import com.codename1.backend.Transactions; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/// A server's mail support: its transport, its queue and its defaults. +/// +/// Application code injects a [MailSender], which sends through the one its +/// server has. This class is what [Mailers#open] answers, for code that sets +/// one up by hand -- a command line tool, a check. +public final class Mailer { + /// How many mails a server with no transport remembers. + private static final int KEPT = 1000; + private static final String HEX = "0123456789abcdef"; + + private final MailTransport transport; + private final MailQueue queue; + private final Address defaultFrom; + private final String domain; + private final boolean capture; + private final boolean warn; + private final AtomicBoolean warned = new AtomicBoolean(); + private final List sent = new ArrayList(); + + Mailer(MailTransport transport, MailQueue queue, Address defaultFrom, String domain, + boolean warnWhenCaptured) { + this.transport = transport; + this.queue = queue; + this.defaultFrom = defaultFrom; + this.domain = domain; + this.capture = transport instanceof MemoryTransport; + this.warn = warnWhenCaptured; + } + + /// The transport mails leave through. + public MailTransport getTransport() { + return transport; + } + + private static String hex(byte[] data) { + StringBuilder sb = new StringBuilder(data.length * 2); + for (byte b : data) { + sb.append(HEX.charAt(b >> 4 & 15)).append(HEX.charAt(b & 15)); + } + return sb.toString(); + } + + private static boolean isHostName(String host) { + for (int i = 0; i < host.length(); i++) { + char c = host.charAt(i); + if (!(c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + || c == '.' || c == '-')) { + return false; + } + } + return host.length() > 0; + } + + private byte[] encode(Mail resolved) throws IOException { + String id = hex(Crypto.randomBytes(16)); + String host = domain != null ? domain : resolved.from.domain(); + if (!isHostName(host)) { + // An address may have a domain outside ASCII; a Message-ID may + // not, and is written a byte a character. + host = "mail.invalid"; + } + return MimeWriter.write(resolved, id + "@" + host, "=_cn1_" + id.substring(0, 24), + System.currentTimeMillis()); + } + + /// Takes `mail` for delivery and returns without waiting for it. + /// + /// Inside a database transaction the mail is sent only if the transaction + /// commits. A failure to deliver is retried and then recorded, never + /// thrown here; what this throws is a mail that cannot be sent at all -- + /// no sender, a template that does not exist. + public void send(Mail mail) throws IOException { + final Mail resolved = mail.resolved(defaultFrom); + byte[] mime = encode(resolved); + if (capture) { + if (Transactions.isActive()) { + Transactions.registerSynchronization(new Transactions.Synchronization() { + @Override + public void afterCommit() { + keep(resolved); + } + }); + } else { + keep(resolved); + } + return; + } + List to = resolved.recipients(); + queue.submit(resolved.from.spec, to.toArray(new String[to.size()]), mime); + } + + /// Takes several mails; they leave over one connection where the + /// transport has connections. + public void send(Mail... mails) throws IOException { + for (Mail m : mails) { + send(m); + } + } + + /// Sends `mail` on the calling thread and returns when the server has + /// taken it, or throws why it did not. No queue, no retry and no regard + /// for a transaction: for a caller that has to know, and has the time to + /// wait. + public void sendNow(Mail mail) throws IOException { + Mail resolved = mail.resolved(defaultFrom); + byte[] mime = encode(resolved); + if (capture) { + keep(resolved); + return; + } + List to = resolved.recipients(); + try { + MailMetrics.transmit(transport, resolved.from.spec, + to.toArray(new String[to.size()]), mime); + } catch (MailException err) { + MailMetrics.failed(err.isPermanent()); + throw err; + } catch (IOException err) { + MailMetrics.failed(false); + throw new MailException("The mail could not be sent: " + err.getMessage(), err); + } + MailMetrics.sent(); + } + + private void keep(Mail resolved) { + if (warn && warned.compareAndSet(false, true)) { + System.err.println("cn1: cn1.mail.url is not set, so mail is kept in memory and " + + "not sent; set it to smtp://..., ses://... or log:"); + } + MailMetrics.sent(); + synchronized (sent) { + if (sent.size() >= KEPT) { + sent.remove(0); + } + sent.add(resolved); + } + } + + /// The mails sent through the `memory:` transport, oldest first: what a + /// test asserts on. Each has its template rendered and its sender filled + /// in. Empty for any other transport. + public List sent() { + synchronized (sent) { + return new ArrayList(sent); + } + } + + /// Forgets what [#sent] answers. + public void clearSent() { + synchronized (sent) { + sent.clear(); + } + } + + /// Mails accepted and neither delivered nor given up on yet. + public int pending() { + return queue.depth(); + } + + /// Whether the transport could reach its server the last time it tried. + public boolean isHealthy() { + return transport.isHealthy(); + } + + /// The name of the transport mail leaves through, such as `smtp`. + public String transportName() { + return transport.getName(); + } + + /// Stops sending. Mails that are due are handed over first, for up to + /// five seconds; with a data source the rest are sent by the next start. + public void close() { + queue.close(5000); + transport.close(); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/Mailers.java b/vm/backend/src/com/codename1/backend/mail/Mailers.java new file mode 100644 index 00000000000..3422ca36699 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/Mailers.java @@ -0,0 +1,250 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.backend.Config; +import com.codename1.backend.DataSource; +import com.codename1.backend.messaging.RetryPolicy; +import com.codename1.impl.backend.BackendAccess; +import com.codename1.impl.backend.MailRuntime; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; + +/// Opens the [Mailer] a configuration names, and finds the one of the calling +/// thread's server. +/// +/// | Setting | Default | | +/// |---|---|---| +/// | `cn1.mail.url` | `memory:` | `smtp://host:587`, `smtps://host:465`, `ses://region`, `log:`, `memory:` | +/// | `cn1.mail.from` | | the sender of a mail that names none: `Shop ` | +/// | `cn1.mail.domain` | the sender's | the domain a Message-ID is made in | +/// | `cn1.mail.username`, `cn1.mail.password` | | credentials, when the URL carries none | +/// | `cn1.mail.retry.max-attempts` | 6 | attempts before a mail is given up on | +/// | `cn1.mail.retry.backoff` | 30000 | milliseconds before the second attempt | +/// | `cn1.mail.retry.multiplier` | 4 | what each later wait is multiplied by | +/// | `cn1.mail.retry.max-backoff` | 3600000 | the longest wait | +/// | `cn1.mail.outbox.sweep` | 10000 | milliseconds between looks for mails a stopped server left | +/// +/// Each transport documents its own settings. Without `cn1.mail.url`, Spring +/// Boot's `spring.mail.host` selects SMTP, with `spring.mail.port`, +/// `spring.mail.username` and `spring.mail.password`. +/// +/// A transport is compiled into an application only when its configuration +/// names it: the build registers that one transport's factory with +/// [#register], so SMTP adds nothing to an application that sends through an +/// API, and neither adds anything to one that sends no mail. +public final class Mailers { + private static final String RESOURCE = "cn1.mail"; + private static final String DATA_SOURCE = "cn1.mail.ds"; + private static final Map FACTORIES = + new HashMap(); + + private Mailers() { + } + + /// Opens a transport for a URL of its scheme. + public interface TransportFactory { + /// Opens the transport `url` names; the caller closes it. + MailTransport open(String url, Config config) throws IOException; + } + + /// Makes `scheme` (`smtp`, `ses`) open through `factory`. Called by the + /// generated wiring, never by application code. + public static void register(String scheme, TransportFactory factory) { + synchronized (FACTORIES) { + FACTORIES.put(scheme, factory); + } + } + + private static final class DataSourceHolder { + DataSource dataSource; + } + + private static DataSourceHolder holder() throws IOException { + return (DataSourceHolder) BackendAccess.get().serverResource(DATA_SOURCE, + new BackendAccess.ResourceFactory() { + @Override + public Object open(Config config) { + return new DataSourceHolder(); + } + }); + } + + /// Gives the calling thread's server's mail support its data source, + /// where the outbox is kept. Called by the generated wiring before any + /// bean can send. + public static void bindDataSource(DataSource dataSource) throws IOException { + DataSourceHolder h = holder(); + synchronized (h) { + h.dataSource = dataSource; + } + } + + /// The mail support of the calling thread's server: opened from its + /// configuration the first time it is asked for, closed when the server + /// stops. + public static Mailer current() throws IOException { + DataSourceHolder h = holder(); + final DataSource ds; + synchronized (h) { + ds = h.dataSource; + } + Object mailer = BackendAccess.get().serverResource(RESOURCE, + new BackendAccess.ResourceFactory() { + @Override + public Object open(Config config) throws IOException { + return new Closer(Mailers.open(config, ds)); + } + }); + return ((Closer) mailer).mailer; + } + + /// What a server keeps: it closes its resources through `close()`. + private static final class Closer implements java.io.Closeable { + final Mailer mailer; + private final MailRuntime.Probe probe; + + Closer(Mailer mailer) { + this.mailer = mailer; + this.probe = new State(mailer); + MailRuntime.opened(probe); + MailMetrics.outbox(); + } + + @Override + public void close() { + MailRuntime.closed(probe); + mailer.close(); + } + } + + /// What the health report and the outbox gauge read of an open mailer. + private static final class State implements MailRuntime.Probe { + private final Mailer mailer; + + State(Mailer mailer) { + this.mailer = mailer; + } + + @Override + public String transport() { + return mailer.transportName(); + } + + @Override + public boolean isHealthy() { + return mailer.isHealthy(); + } + + @Override + public int pending() { + return mailer.pending(); + } + } + + /// Whether `config` names a transport that sends. + public static boolean isConfigured(Config config) throws IOException { + String url = url(config); + return url != null && !url.startsWith("memory:"); + } + + /// The URL `config` names, or null when it names none. + public static String url(Config config) throws IOException { + if (config == null) { + return null; + } + String url = config.get("cn1.mail.url"); + if (url != null && url.trim().length() > 0) { + return url.trim(); + } + String host = config.get("spring.mail.host"); + if (host != null && host.trim().length() > 0) { + int port = config.getInt("spring.mail.port", 25); + return (port == 465 ? "smtps://" : "smtp://") + host.trim() + ":" + port; + } + return null; + } + + /// Opens the mail support `config` names; the caller closes it. + /// + /// - `dataSource`: the server's, or null; where the outbox is kept + public static Mailer open(Config config, DataSource dataSource) throws IOException { + String url = url(config); + boolean unset = url == null; + if (unset) { + url = "memory:"; + } + MailTransport transport; // NOPMD CloseResource - owned by the Mailer returned + if (url.startsWith("memory:")) { + transport = new MemoryTransport(); + } else if (url.startsWith("log:")) { + transport = new LogTransport(); + } else { + int colon = url.indexOf(':'); + String scheme = colon < 0 ? url : url.substring(0, colon); + TransportFactory f; + synchronized (FACTORIES) { + f = FACTORIES.get(scheme); + if (f == null && scheme.endsWith("s")) { + // smtps: the TLS form of the same transport. + f = FACTORIES.get(scheme.substring(0, scheme.length() - 1)); + } + } + if (f == null) { + throw new IOException("cn1.mail.url \"" + scheme + ":...\" names a transport " + + "this application was not built with; set cn1.mail.url in " + + "application.properties so the build includes it"); + } + transport = f.open(url, config); + } + String from = config == null ? null : config.get("cn1.mail.from"); + Address sender = null; + if (from != null && from.trim().length() > 0) { + try { + sender = Address.parse(from); + } catch (IllegalArgumentException err) { + transport.close(); + throw new IOException("cn1.mail.from: " + err.getMessage(), err); + } + } + String domain = config == null ? null : config.get("cn1.mail.domain"); + if (domain != null && domain.trim().length() == 0) { + domain = null; + } + RetryPolicy retry = new RetryPolicy( + config == null ? 6 : config.getInt("cn1.mail.retry.max-attempts", 6), + config == null ? 30000 : config.getInt("cn1.mail.retry.backoff", 30000), + config == null ? 4 : config.getInt("cn1.mail.retry.multiplier", 4), + config == null ? 3600000 : config.getInt("cn1.mail.retry.max-backoff", 3600000)); + // A transport that sends nothing gets no outbox. Every server on a + // database sweeps the one table, and one with no mail server -- a + // developer's, a test run -- would take the rows of the others, call + // them sent and delete them. + boolean sends = !(transport instanceof MemoryTransport + || transport instanceof LogTransport); + MailQueue queue = new MailQueue(transport, sends ? dataSource : null, retry, + config == null ? 10000 : config.getInt("cn1.mail.outbox.sweep", 10000)); + return new Mailer(transport, queue, sender, domain == null ? null : domain.trim(), unset); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MemoryTransport.java b/vm/backend/src/com/codename1/backend/mail/MemoryTransport.java new file mode 100644 index 00000000000..154e8a0c7ab --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MemoryTransport.java @@ -0,0 +1,43 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +/// Sends nothing: the transport of a server with no `cn1.mail.url`, and of +/// every test. The server's [Mailer] keeps what was sent through it, for +/// [MailSender#sent] to answer. +final class MemoryTransport extends MailTransport { + @Override + public String getName() { + return "memory"; + } + + @Override + public void send(String from, String[] recipients, byte[] mime) { + // Kept by the Mailer, as a Mail, which is what a test asserts on. + } + + @Override + public void close() { + // Holds nothing. + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/MimeWriter.java b/vm/backend/src/com/codename1/backend/mail/MimeWriter.java new file mode 100644 index 00000000000..b2a6cba6189 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/MimeWriter.java @@ -0,0 +1,559 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail; + +import com.codename1.backend.ByteSink; +import com.codename1.backend.Http1Date; + +/// Writes a [Mail] as the bytes a mail server takes: RFC 5322 headers and a +/// MIME body. +/// +/// The message is encoded once, into one buffer, and those bytes are what +/// every attempt to send it uses. Nothing here builds a String of the body: +/// text is turned to UTF-8 once and encoded from the bytes, an attachment is +/// encoded from its array straight into the output. +/// +/// The structure is the smallest that holds the mail: +/// +/// - text alone is one `text/plain` part +/// - text and HTML are `multipart/alternative` +/// - HTML with inline images is `multipart/related` in the HTML's place +/// - attachments wrap all of it in `multipart/mixed` +/// +/// A text body goes as `7bit` when it is ASCII in lines of at most 78 +/// characters, and as quoted-printable otherwise; everything else is Base64 +/// in 76 character lines. A header that is not ASCII is written as RFC 2047 +/// encoded words, and every header is folded at 78 characters. +/// +/// Line endings are CRLF throughout and no line begins a dot-stuffed pair: +/// doubling a leading dot is SMTP's transparency, done by the SMTP transport +/// while it copies these bytes, since a provider's HTTP API takes them as they +/// are. +final class MimeWriter { + private static final String BASE64 = + "ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789+/"; + private static final String HEX = "0123456789ABCDEF"; + /// The longest line a header is folded to, without its CRLF. + private static final int FOLD = 78; + /// The longest line that may hold an encoded word. + private static final int ENCODED_LINE = 76; + + private final ByteSink out; + private final ByteSink scratch = new ByteSink(256); + private int col; + + private MimeWriter(ByteSink out) { + this.out = out; + } + + /// Encodes `mail`, which must be resolved: it has a sender and no + /// template left to render. + /// + /// - `messageId`: without the angle brackets + /// - `boundary`: a string that cannot occur in the mail, which random + /// bytes make certain enough; each level adds a letter to it + /// - `millis`: the time for the Date header + static byte[] write(Mail mail, String messageId, String boundary, long millis) { + int size = 1024; + if (mail.text != null) { + size += mail.text.length() * 2; + } + if (mail.html != null) { + size += mail.html.length() * 2; + } + for (Mail.Part p : mail.attachments) { + size += p.data.length / 3 * 4 + p.data.length / 28 + 512; + } + for (Mail.Part p : mail.inlines) { + size += p.data.length / 3 * 4 + p.data.length / 28 + 512; + } + ByteSink sink = new ByteSink(size); + new MimeWriter(sink).message(mail, messageId, boundary, millis); + byte[] exact = new byte[sink.length()]; + System.arraycopy(sink.bytes(), 0, exact, 0, exact.length); + return exact; + } + + private void message(Mail m, String messageId, String boundary, long millis) { + String date = Http1Date.format(millis); + // "Sun, 06 Nov 1994 08:49:37 GMT": RFC 5322 wants a numeric zone. + raw("Date: " + date.substring(0, date.length() - 3) + "+0000"); + addresses("From", new Address[] {m.from}); + if (m.replyTo != null) { + addresses("Reply-To", new Address[] {m.replyTo}); + } + if (m.to.length > 0) { + addresses("To", m.to); + } + if (m.cc.length > 0) { + addresses("Cc", m.cc); + } + // Bcc is in the envelope only: writing it would tell every recipient. + if (m.subject != null) { + text("Subject", m.subject); + } + raw("Message-ID: <" + messageId + ">"); + raw("MIME-Version: 1.0"); + for (int i = 0; i < m.headers.length; i += 2) { + text(m.headers[i], m.headers[i + 1]); + } + if (m.attachments.length == 0) { + content(m, boundary); + return; + } + String b = boundary + "m"; + multipart("mixed", null, b); + open(b); + content(m, boundary); + for (Mail.Part p : m.attachments) { + open(b); + part(p); + } + close(b); + } + + /// The text, the HTML, or both as alternatives. + private void content(Mail m, String boundary) { + if (m.html == null) { + textPart("plain", m.text == null ? "" : m.text); + return; + } + if (m.text == null) { + htmlContent(m, boundary); + return; + } + String b = boundary + "a"; + multipart("alternative", null, b); + open(b); + textPart("plain", m.text); + // The richest form goes last: a reader shows the last one it can. + open(b); + htmlContent(m, boundary); + close(b); + } + + /// The HTML, with the images it shows beside it when it has any. + private void htmlContent(Mail m, String boundary) { + if (m.inlines.length == 0) { + textPart("html", m.html); + return; + } + String b = boundary + "r"; + multipart("related", "text/html", b); + open(b); + textPart("html", m.html); + for (Mail.Part p : m.inlines) { + open(b); + part(p); + } + close(b); + } + + private void multipart(String subtype, String type, String boundary) { + out.putAscii("Content-Type: multipart/"); + out.putAscii(subtype); + out.putAscii(";\r\n"); + if (type != null) { + out.putAscii(" type=\""); + out.putAscii(type); + out.putAscii("\";\r\n"); + } + out.putAscii(" boundary=\""); + out.putAscii(boundary); + out.putAscii("\"\r\n\r\n"); + } + + /// Every part ends with a CRLF, which is the one the delimiter owns. + private void open(String boundary) { + out.putAscii("--"); + out.putAscii(boundary); + out.putAscii("\r\n"); + } + + private void close(String boundary) { + out.putAscii("--"); + out.putAscii(boundary); + out.putAscii("--\r\n"); + } + + private void raw(String line) { + out.putAscii(line); + out.putAscii("\r\n"); + } + + // ---- headers ---- + + private void begin(String name) { + out.putAscii(name); + out.putAscii(": "); + col = name.length() + 2; + } + + private void fold() { + out.putAscii("\r\n "); + col = 1; + } + + /// Writes one unbreakable piece of a header, on a new line when it does + /// not fit this one. `space` says whether whitespace separates it from + /// what came before, which is the only place a header may be folded. + private void token(String value, boolean space, boolean ascii) { + if (space) { + if (col + 1 + value.length() > FOLD) { + fold(); + } else { + out.put(' '); + col++; + } + } + if (ascii) { + out.putAscii(value); + } else { + out.putUtf8(value); + } + col += value.length(); + } + + private static boolean needsEncoding(String value) { + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c < ' ' || c > 126) { + return true; + } + } + // ASCII that looks like an encoded word would be decoded as one. + return value.indexOf("=?") >= 0; + } + + /// Whether `value` has a run without a space that no folding could keep + /// inside the 998 bytes a line may be; a server refuses a longer one. + static boolean longWord(String value) { + int run = 0; + for (int i = 0; i < value.length(); i++) { + run = value.charAt(i) == ' ' ? 0 : run + 1; + if (run > 900) { + return true; + } + } + return false; + } + + /// An unstructured header: Subject, or one of the application's own. + private void text(String name, String value) { + begin(name); + if (needsEncoding(value) || longWord(value)) { + encoded(value, false); + } else { + int from = 0; + boolean first = true; + while (from <= value.length()) { + int space = value.indexOf(' ', from); + if (space < 0) { + space = value.length(); + } + token(value.substring(from, space), !first, true); + first = false; + from = space + 1; + } + } + out.putAscii("\r\n"); + } + + private void addresses(String name, Address[] list) { + begin(name); + for (int i = 0; i < list.length; i++) { + Address a = list[i]; + String tail = i < list.length - 1 ? "," : ""; + boolean space = i > 0; + if (a.name == null) { + token(a.spec + tail, space, a.ascii); + continue; + } + // A name is one piece, spaces and all, so a long one is written + // as encoded words, which fold. + if (needsEncoding(a.name) || a.name.length() > 60) { + encoded(a.name, space); + } else { + token(phrase(a.name), space, true); + } + token("<" + a.spec + ">" + tail, true, a.ascii); + } + out.putAscii("\r\n"); + } + + /// A display name as it may stand in a header: as it is when it is words, + /// and as a quoted string when it holds a character with a meaning there. + private static String phrase(String name) { + boolean plain = true; + for (int i = 0; plain && i < name.length(); i++) { + char c = name.charAt(i); + plain = c >= 'a' && c <= 'z' || c >= 'A' && c <= 'Z' || c >= '0' && c <= '9' + || c == ' ' || c == '-' || c == '_' || c == '\'' || c == '!' || c == '+'; + } + return plain ? name : quoted(name); + } + + private static String quoted(String value) { + StringBuilder sb = new StringBuilder(value.length() + 4); + sb.append('"'); + for (int i = 0; i < value.length(); i++) { + char c = value.charAt(i); + if (c == '"' || c == '\\') { + sb.append('\\'); + } + sb.append(c); + } + return sb.append('"').toString(); + } + + /// Writes `value` as RFC 2047 encoded words, each `=?UTF-8?B?...?=`, as + /// many as it takes: an encoded word may be 75 characters and the line it + /// is on 76, and a character's bytes may not be split between two words. + private void encoded(String value, boolean space) { + int i = 0; + int n = value.length(); + boolean lead = space; + while (i < n) { + int budget = (ENCODED_LINE - col - (lead ? 1 : 0) - 12) / 4 * 3; + if (budget < 4) { + fold(); + budget = (ENCODED_LINE - col - 12) / 4 * 3; + } else if (lead) { + out.put(' '); + col++; + } + scratch.reset(); + while (i < n) { + int cp = value.charAt(i); + int chars = 1; + if (cp >= 0xD800 && cp <= 0xDBFF && i + 1 < n && value.charAt(i + 1) >= 0xDC00 + && value.charAt(i + 1) <= 0xDFFF) { + cp = 0x10000 + ((cp - 0xD800) << 10) + (value.charAt(i + 1) - 0xDC00); + chars = 2; + } else if (cp >= 0xD800 && cp <= 0xDFFF) { + // Half a pair is not a character; UTF-8 has no form for it. + cp = '?'; + } + int bytes = cp < 0x80 ? 1 : cp < 0x800 ? 2 : cp < 0x10000 ? 3 : 4; + if (scratch.length() + bytes > budget) { + break; + } + scratch.putCodePoint(cp); + i += chars; + } + out.putAscii("=?UTF-8?B?"); + base64(scratch.bytes(), 0, scratch.length(), false); + out.putAscii("?="); + col += 12 + (scratch.length() + 2) / 3 * 4; + // Whitespace between two encoded words is not part of the text. + lead = true; + } + } + + // ---- parts ---- + + private void textPart(String subtype, String value) { + scratch.reset(); + scratch.putUtf8(value); + byte[] data = scratch.bytes(); + int length = scratch.length(); + boolean sevenBit = true; + int line = 0; + for (int i = 0; sevenBit && i < length; i++) { + int b = data[i]; + if (b == '\n' || b == '\r') { + line = 0; + } else { + line++; + sevenBit = b >= 0 && (b >= ' ' || b == '\t') && b != 127 && line <= FOLD; + } + } + out.putAscii("Content-Type: text/"); + out.putAscii(subtype); + out.putAscii("; charset=UTF-8\r\nContent-Transfer-Encoding: "); + out.putAscii(sevenBit ? "7bit\r\n\r\n" : "quoted-printable\r\n\r\n"); + if (sevenBit) { + plain(data, length); + } else { + quotedPrintable(data, length); + } + } + + /// Whether a line break starts at `i`; a CRLF, a lone LF and a lone CR + /// are each one break. + private static int lineBreak(byte[] data, int i, int length) { + if (data[i] == '\n') { + return 1; + } + if (data[i] == '\r') { + return i + 1 < length && data[i + 1] == '\n' ? 2 : 1; + } + return 0; + } + + private void plain(byte[] data, int length) { + int i = 0; + boolean ended = true; + while (i < length) { + int br = lineBreak(data, i, length); + if (br > 0) { + out.putAscii("\r\n"); + i += br; + ended = true; + } else { + out.put(data[i++]); + ended = false; + } + } + if (!ended) { + out.putAscii("\r\n"); + } + } + + private void quotedPrintable(byte[] data, int length) { + int i = 0; + int width = 0; + boolean ended = true; + while (i < length) { + int br = lineBreak(data, i, length); + if (br > 0) { + out.putAscii("\r\n"); + i += br; + width = 0; + ended = true; + continue; + } + int b = data[i] & 0xff; + boolean literal = b > ' ' && b < 127 && b != '='; + if (b == ' ' || b == '\t') { + // Whitespace at the end of a line is removed in transit unless + // it is encoded. + literal = i + 1 < length && lineBreak(data, i + 1, length) == 0; + } + int need = literal ? 1 : 3; + // 76 with the soft break's own "=". + if (width + need > 75) { + out.putAscii("=\r\n"); + width = 0; + } + if (literal) { + out.put(b); + } else { + out.put('='); + out.put(HEX.charAt(b >> 4)); + out.put(HEX.charAt(b & 15)); + } + width += need; + ended = false; + i++; + } + if (!ended) { + out.putAscii("\r\n"); + } + } + + private void part(Mail.Part p) { + boolean asciiName = p.name != null && !needsEncoding(p.name); + out.putAscii("Content-Type: "); + out.putAscii(p.type); + if (asciiName) { + out.putAscii(";\r\n name="); + out.putAscii(quoted(p.name)); + } + out.putAscii("\r\nContent-Transfer-Encoding: base64\r\n"); + if (p.contentId != null) { + out.putAscii("Content-ID: <"); + out.putAscii(p.contentId); + out.putAscii(">\r\nContent-Disposition: inline\r\n\r\n"); + } else { + out.putAscii("Content-Disposition: attachment;\r\n filename"); + if (asciiName) { + out.put('='); + out.putAscii(quoted(p.name)); + } else { + // RFC 2231: the only form of a parameter that says which + // charset its bytes are in. + out.putAscii("*=UTF-8''"); + scratch.reset(); + scratch.putUtf8(p.name); + byte[] raw = scratch.bytes(); + for (int i = 0; i < scratch.length(); i++) { + int b = raw[i] & 0xff; + if (b >= 'a' && b <= 'z' || b >= 'A' && b <= 'Z' || b >= '0' && b <= '9' + || b == '.' || b == '-' || b == '_') { + out.put(b); + } else { + out.put('%'); + out.put(HEX.charAt(b >> 4)); + out.put(HEX.charAt(b & 15)); + } + } + } + out.putAscii("\r\n\r\n"); + } + base64(p.data, 0, p.data.length, true); + } + + /// Base64 of `data` into the output; `wrap` breaks it into 76 character + /// lines and ends it with a line break, which a body needs and an encoded + /// word must not have. + private void base64(byte[] data, int offset, int length, boolean wrap) { + out.ensure(length / 3 * 4 + length / 28 + 8); + int end = offset + length; + int i = offset; + int width = 0; + while (i + 2 < end) { + int v = (data[i] & 0xff) << 16 | (data[i + 1] & 0xff) << 8 | data[i + 2] & 0xff; + out.put(BASE64.charAt(v >> 18)); + out.put(BASE64.charAt(v >> 12 & 63)); + out.put(BASE64.charAt(v >> 6 & 63)); + out.put(BASE64.charAt(v & 63)); + i += 3; + width += 4; + if (wrap && width == 76) { + out.put('\r'); + out.put('\n'); + width = 0; + } + } + if (end - i == 1) { + int v = (data[i] & 0xff) << 16; + out.put(BASE64.charAt(v >> 18)); + out.put(BASE64.charAt(v >> 12 & 63)); + out.put('='); + out.put('='); + width += 4; + } else if (end - i == 2) { + int v = (data[i] & 0xff) << 16 | (data[i + 1] & 0xff) << 8; + out.put(BASE64.charAt(v >> 18)); + out.put(BASE64.charAt(v >> 12 & 63)); + out.put(BASE64.charAt(v >> 6 & 63)); + out.put('='); + width += 4; + } + if (wrap && width > 0) { + out.put('\r'); + out.put('\n'); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/package-info.java b/vm/backend/src/com/codename1/backend/mail/package-info.java new file mode 100644 index 00000000000..2e3b5c57860 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/package-info.java @@ -0,0 +1,34 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// Sending email: a [com.codename1.backend.mail.Mail] is built, a +/// [com.codename1.backend.mail.MailSender] takes it, and the server delivers +/// it in the background over the transport `cn1.mail.url` names. +/// +/// Mail sent inside a database transaction leaves only if the transaction +/// commits. A mail the server could not hand over is tried again; one it was +/// refused is recorded and not sent twice. With no transport configured, mail +/// is kept in memory, where a test reads it back. +/// +/// Receiving mail, signing and encrypting it, and character sets other than +/// UTF-8 are not part of this package. +package com.codename1.backend.mail; diff --git a/vm/backend/src/com/codename1/backend/mail/ses/SesTransport.java b/vm/backend/src/com/codename1/backend/mail/ses/SesTransport.java new file mode 100644 index 00000000000..1f326b68af3 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/ses/SesTransport.java @@ -0,0 +1,246 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail.ses; + +import com.codename1.backend.Base64; +import com.codename1.backend.Config; +import com.codename1.backend.Json; +import com.codename1.backend.Web; +import com.codename1.backend.aws.Aws; +import com.codename1.backend.aws.Credentials; +import com.codename1.backend.mail.MailAuthenticationException; +import com.codename1.backend.mail.MailException; +import com.codename1.backend.mail.MailSendException; +import com.codename1.backend.mail.MailTransport; +import com.codename1.backend.mail.Mailers; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; +import java.util.concurrent.atomic.AtomicBoolean; + +/// Sends mail through Amazon SES, by its API rather than its SMTP endpoint: +/// one signed HTTPS request a mail, and no SMTP credentials to issue. +/// +/// ```properties +/// cn1.mail.url=ses://eu-west-1 +/// ``` +/// +/// Credentials are the machine's own -- the environment, a container role or +/// an instance role -- found as for every other AWS service here. The sender +/// has to be an identity SES has verified. +/// +/// | Setting | Default | | +/// |---|---|---| +/// | `cn1.mail.ses.configuration-set` | | the configuration set a mail is sent with | +/// | `cn1.mail.ses.region` | `us-east-1` | the region to sign for, when the URL names a host and not a region | +/// +/// The mail is handed over as it was encoded, so what arrives is what an SMTP +/// server would have been given. SES takes at most 50 recipients a request; a +/// mail to more is sent in several. `ses://localhost:4566` names an endpoint +/// instead of a region, for a stand-in on the same machine. +public final class SesTransport extends MailTransport { + /// What SES accepts in one request. + private static final int MAX_RECIPIENTS = 50; + + private final String host; + private final String region; + private final boolean secure; + private final String configurationSet; + private final Object credentialLock = new Object(); + private Credentials credentials; + private final Credentials fixed; + private final AtomicBoolean healthy = new AtomicBoolean(true); + private final AtomicBoolean closed = new AtomicBoolean(); + + /// Opens `ses:` URLs; what the generated wiring registers. + public static final class Factory implements Mailers.TransportFactory { + @Override + public MailTransport open(String url, Config config) throws IOException { + return new SesTransport(url, + config == null ? null : config.get("cn1.mail.ses.region"), + config == null ? null : config.get("cn1.mail.ses.configuration-set")); + } + } + + /// A transport for `url`. + /// + /// - `region`: the region to sign for when `url` names a host; may be null + /// - `configurationSet`: the configuration set to send with, or null + public SesTransport(String url, String region, String configurationSet) throws IOException { + this(url, region, configurationSet, null); + } + + /// A transport for `url` that signs with `fixed` rather than with what the + /// environment, the container or the instance provides -- for an account + /// other than the one the server runs as. Credentials given this way are + /// never refreshed, so they should not be ones that expire. + /// + /// - `region`: the region to sign for when `url` names a host; may be null + /// - `configurationSet`: the configuration set to send with, or null + /// - `fixed`: the credentials to sign with, or null to resolve them + public SesTransport(String url, String region, String configurationSet, Credentials fixed) + throws IOException { + this.fixed = fixed; + int scheme = url.indexOf("://"); + if (scheme < 0 || !"ses".equalsIgnoreCase(url.substring(0, scheme))) { + throw new IOException("Not an ses:// URL"); + } + String rest = url.substring(scheme + 3); + int slash = rest.indexOf('/'); + if (slash >= 0) { + rest = rest.substring(0, slash); + } + if (rest.length() == 0) { + throw new IOException("cn1.mail.url names no region; ses://eu-west-1 does"); + } + if (rest.indexOf(':') < 0 && rest.indexOf('.') < 0) { + this.region = rest; + this.host = "email." + rest + ".amazonaws.com"; + this.secure = true; + } else { + this.region = region == null || region.trim().length() == 0 ? "us-east-1" + : region.trim(); + this.host = rest; + String name = rest.indexOf(':') < 0 ? rest : rest.substring(0, rest.lastIndexOf(':')); + this.secure = !"localhost".equalsIgnoreCase(name) && !"127.0.0.1".equals(name) // NOPMD AvoidUsingHardCodedIP - loopback is the one place plain HTTP is allowed + && !"[::1]".equals(name); + } + this.configurationSet = configurationSet == null + || configurationSet.trim().length() == 0 ? null : configurationSet.trim(); + } + + @Override + public String getName() { + return "ses"; + } + + private Credentials credentials() throws IOException { + if (fixed != null) { + return fixed; + } + synchronized (credentialLock) { + if (credentials == null || credentials.isExpiring(60000)) { + credentials = Credentials.resolve(); + } + return credentials; + } + } + + @Override + public void send(String from, String[] recipients, byte[] mime) throws IOException { + if (closed.get()) { + throw new MailException("The mail transport is closed"); + } + String data = Base64.encode(mime); + for (int at = 0; at < recipients.length; at += MAX_RECIPIENTS) { + List to = new ArrayList(); + for (int i = at; i < recipients.length && i < at + MAX_RECIPIENTS; i++) { + to.add(recipients[i]); + } + try { + post(from, to, data); + } catch (MailSendException err) { + if (at > 0 && !err.isPermanent()) { + // Some recipients have the mail already; trying it again + // whole would send it to them twice. + MailSendException partial = new MailSendException("The mail was sent to " + + at + " of " + recipients.length + " recipients: " + + err.getMessage(), err.getCode(), true); + partial.initCause(err); + throw partial; + } + throw err; + } catch (IOException err) { + if (at > 0) { + // The same, for a request that never got its answer. + MailSendException partial = new MailSendException("The mail was sent to " + + at + " of " + recipients.length + " recipients: " + + err.getMessage(), 0, true); + partial.initCause(err); + throw partial; + } + throw err; + } + } + } + + private void post(String from, List to, String data) throws IOException { + Map destination = new LinkedHashMap(); + destination.put("ToAddresses", to); + Map raw = new LinkedHashMap(); + raw.put("Data", data); + Map content = new LinkedHashMap(); + content.put("Raw", raw); + Map request = new LinkedHashMap(); + request.put("FromEmailAddress", from); + request.put("Destination", destination); + request.put("Content", content); + if (configurationSet != null) { + request.put("ConfigurationSetName", configurationSet); + } + Map headers = new LinkedHashMap(); + headers.put("content-type", "application/json"); + Web.Result r; + try { + r = Aws.send(credentials(), region, "ses", "POST", host, + "/v2/email/outbound-emails", null, headers, + WireBuffer.utf8(Json.write(request)), null, secure); + } catch (IOException err) { + healthy.set(false); + throw err; + } + int status = r.getStatus(); + if (r.isSuccess()) { + healthy.set(true); + return; + } + if (status <= 0) { + healthy.set(false); + throw new MailException("SES could not be reached: " + r.getError()); + } + healthy.set(status < 500); + String body = r.getBodyAsString(); + String why = "SES refused the mail (" + status + "): " + + (body == null ? r.getError() : body.length() > 300 ? body.substring(0, 300) + : body); + if (status == 401 || status == 403) { + throw new MailAuthenticationException(why, status); + } + // Too many requests and a request that timed out are the two refusals + // of the client's own that the same mail can get past later. + throw new MailSendException(why, status, status < 500 && status != 429 && status != 408); + } + + @Override + public boolean isHealthy() { + return healthy.get(); + } + + @Override + public void close() { + closed.set(true); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/ses/package-info.java b/vm/backend/src/com/codename1/backend/mail/ses/package-info.java new file mode 100644 index 00000000000..34d00d1b0c6 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/ses/package-info.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// The Amazon SES transport: [com.codename1.backend.mail.ses.SesTransport] +/// hands a mail to the SES API over HTTPS, signed with the credentials of the +/// machine it runs on. +package com.codename1.backend.mail.ses; diff --git a/vm/backend/src/com/codename1/backend/mail/smtp/SmtpConnection.java b/vm/backend/src/com/codename1/backend/mail/smtp/SmtpConnection.java new file mode 100644 index 00000000000..6a66d8cfcce --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/smtp/SmtpConnection.java @@ -0,0 +1,560 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail.smtp; + +import com.codename1.backend.Base64; +import com.codename1.backend.Tcp; +import com.codename1.backend.mail.MailAuthenticationException; +import com.codename1.backend.mail.MailException; +import com.codename1.backend.mail.MailSendException; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; + +/// One session with an SMTP server: RFC 5321 with the extensions a mail +/// provider expects of a client today. +/// +/// - **STARTTLS** (RFC 3207). The connection is upgraded where it stands and +/// the greeting repeated, since nothing learned before the upgrade can be +/// trusted. Bytes that arrive between the server's go-ahead and the +/// handshake are an injection -- a response planted in the clear to be read +/// as if it came over TLS -- and end the session. +/// - **AUTH** `PLAIN`, `LOGIN` and `XOAUTH2`, and only over TLS. +/// - **PIPELINING** (RFC 2920): the sender, every recipient and `DATA` leave +/// in one write and their replies are read in order, which makes a mail two +/// round trips rather than three plus one per recipient. +/// - **SIZE**: a mail larger than the server accepts is refused before a byte +/// of it is sent. +/// - **SMTPUTF8** (RFC 6531), asked for only by a mail with an address that +/// needs it. +/// +/// Not thread safe; [SmtpTransport] lends one to a thread at a time. +final class SmtpConnection { + /// The most staged before a write: the body leaves in pieces this size + /// rather than being copied whole into the buffer first. + private static final int CHUNK = 32768; + + private final Tcp tcp; + private WireBuffer wire; + private boolean broken; + private boolean pipelining; + private boolean startTls; + private boolean smtpUtf8; + private boolean sizeOffered; + private long maxSize; + private String mechanisms = ""; + /// More lines than any server's reply has; an EHLO lists a dozen or two. + private static final int MAX_REPLY_LINES = 100; + private long lastUsed; + // One reply at a time, emptied before each and capped per line where it is + // filled: kept so reading a reply allocates nothing. + private final StringBuilder text = new StringBuilder(128); // NOPMD AvoidStringBufferField - bounded, see above + /// The lines of the last reply, each without its code. + private final List lines = new ArrayList(); + + private SmtpConnection(Tcp tcp) { + this.tcp = tcp; + this.wire = new WireBuffer(tcp); + } + + /// Connects, secures the connection and signs in. + /// + /// - `implicitTls`: TLS from the first byte, as port 465 speaks it + /// - `requireTls`: fail when the server offers no STARTTLS, rather than + /// carry on in the clear + /// - `caFile`: a certificate file to trust instead of the system's, or null + /// - `helo`: the name this client gives for itself + /// - `mechanism`: `plain`, `login`, `xoauth2`, or null to take the first + /// of plain and login the server offers + static SmtpConnection open(String host, int port, boolean implicitTls, boolean requireTls, + String caFile, String helo, String user, String password, + String mechanism, int connectTimeoutMillis, + int readTimeoutMillis) throws IOException { + Tcp tcp = Tcp.connect(host, port, connectTimeoutMillis); + try { + if (readTimeoutMillis > 0) { + tcp.setReadTimeout(readTimeoutMillis); + } + if (implicitTls) { + secure(tcp, host, caFile); + } + SmtpConnection c = new SmtpConnection(tcp); + c.expect(c.reply(), 220, "greeting"); + c.ehlo(helo); + if (!tcp.isSecure()) { + if (c.startTls) { + c.command("STARTTLS"); + c.expect(c.reply(), 220, "STARTTLS"); + if (c.wire.hasBuffered()) { + throw new MailException("The mail server sent data after agreeing to " + + "STARTTLS and before the handshake; that is a response " + + "injected in the clear, and the connection was dropped"); + } + secure(tcp, host, caFile); + c.wire = new WireBuffer(tcp); + c.ehlo(helo); + } else if (requireTls) { + throw new MailException("The mail server at " + host + ":" + port + + " offers no STARTTLS. Use smtps:// for a port that speaks TLS " + + "from the start, or set cn1.mail.smtp.starttls=false to send in " + + "the clear"); + } + } + if (user != null) { + c.authenticate(user, password == null ? "" : password, mechanism); + } + c.lastUsed = System.currentTimeMillis(); + return c; + } catch (IOException err) { + tcp.close(); + throw err; + } catch (RuntimeException err) { + tcp.close(); + throw new IOException("Malformed reply from the mail server: " + err, err); + } + } + + private static void secure(Tcp tcp, String host, String caFile) throws IOException { + if (caFile != null) { + tcp.startTls(host, caFile); + } else { + tcp.startTls(host); + } + } + + boolean isBroken() { + return broken; + } + + /// When this connection last finished an exchange. + long lastUsed() { + return lastUsed; + } + + private void command(String line) throws IOException { + stage(line); + flush(); + } + + private void stage(String line) { + wire.writeBytes(WireBuffer.utf8(line)); + wire.writeByte('\r'); + wire.writeByte('\n'); + } + + private void flush() throws IOException { + try { + wire.flush(); + } catch (IOException err) { + broken = true; + throw err; + } + } + + /// Reads one reply, of however many lines, and answers its code; its text + /// is left in [#text]. + private int reply() throws IOException { + text.setLength(0); + lines.clear(); + try { + while (true) { + String line = wire.readLine(); + if (lines.size() >= MAX_REPLY_LINES) { + // A server that never ends its reply would hold the one + // thread every mail leaves on. + throw new IOException("The mail server's reply has no end"); + } + // Whole, for the extensions an EHLO lists: the text below is + // capped, and an extension past the cap would read as one + // the server does not have. + lines.add(line.length() > 4 ? line.substring(4) : ""); + if (line.length() < 3) { + throw new IOException("Malformed reply from the mail server"); + } + int code = 0; + for (int i = 0; i < 3; i++) { + char c = line.charAt(i); + if (c < '0' || c > '9') { + throw new IOException("Malformed reply from the mail server"); + } + code = code * 10 + (c - '0'); + } + if (line.length() > 4) { + if (text.length() > 0) { + text.append('\n'); + } + // A reply is the server's to word: what is quoted from it + // is capped, so it cannot fill a log or a table. + if (text.length() < 400) { + text.append(line, 4, Math.min(line.length(), 204)); + } + } + if (line.length() == 3 || line.charAt(3) == ' ') { + return code; + } + if (line.charAt(3) != '-') { + throw new IOException("Malformed reply from the mail server"); + } + } + } catch (IOException err) { + broken = true; + throw err; + } + } + + private MailSendException refused(String what, int code) { + return new MailSendException("The mail server refused " + what + ": " + code + " " + + firstLine(), code, code >= 500); + } + + private String firstLine() { + int nl = text.indexOf("\n"); + return nl < 0 ? text.toString() : text.substring(0, nl); + } + + /// For the replies before any mail is named -- the greeting, EHLO, + /// STARTTLS. A refusal there says the server is not taking mail now: a + /// 554 from one under maintenance, or one that has listed this host. + /// It is the server that is refused, not the mail, so the mail is tried + /// again, where a 5xx to the mail itself is final. + private void expect(int code, int wanted, String what) throws IOException { + if (code != wanted) { + broken = true; + throw new MailSendException("The mail server refused " + what + ": " + code + " " + + firstLine(), code, false); + } + } + + private void ehlo(String helo) throws IOException { + command("EHLO " + helo); + int code = reply(); + expect(code, 250, "EHLO"); + pipelining = false; + startTls = false; + smtpUtf8 = false; + sizeOffered = false; + maxSize = 0; + mechanisms = ""; + // The first line is the server's name; each one after it an extension. + for (int i = 1; i < lines.size(); i++) { + extension(lines.get(i)); + } + } + + private static boolean keyword(String line, String word) { + return line.regionMatches(true, 0, word, 0, word.length()) + && (line.length() == word.length() || line.charAt(word.length()) == ' ' + || line.charAt(word.length()) == '='); + } + + private void extension(String line) { + if (keyword(line, "PIPELINING")) { + pipelining = true; + } else if (keyword(line, "STARTTLS")) { + startTls = true; + } else if (keyword(line, "SMTPUTF8")) { + smtpUtf8 = true; + } else if (keyword(line, "SIZE")) { + sizeOffered = true; + long n = 0; + for (int i = 5; i < line.length(); i++) { + char c = line.charAt(i); + if (c < '0' || c > '9' || n > Long.MAX_VALUE / 20) { + break; + } + n = n * 10 + (c - '0'); + } + maxSize = n; + } else if (keyword(line, "AUTH")) { + mechanisms = " " + line.substring(4).replace('=', ' ') + " "; + } + } + + private boolean offers(String mechanism) { + String wanted = " " + mechanism + " "; + for (int i = 0; i + wanted.length() <= mechanisms.length(); i++) { + if (mechanisms.regionMatches(true, i, wanted, 0, wanted.length())) { + return true; + } + } + return false; + } + + private void authenticate(String user, String password, String mechanism) + throws IOException { + if (!tcp.isSecure()) { + throw new MailAuthenticationException("The connection to the mail server is not " + + "encrypted, and a password is not sent over one that is not. Use " + + "smtps://, or a server that offers STARTTLS", 0); + } + String chosen = mechanism; + if (chosen == null) { + chosen = offers("PLAIN") ? "plain" : offers("LOGIN") ? "login" : null; + if (chosen == null) { + throw new MailAuthenticationException("The mail server offers none of the " + + "sign-in mechanisms PLAIN and LOGIN; set cn1.mail.smtp.auth=xoauth2 " + + "when the password is an OAuth 2 access token", 0); + } + } + int code; + if ("xoauth2".equalsIgnoreCase(chosen)) { + command("AUTH XOAUTH2 " + Base64.encode(WireBuffer.utf8("user=" + user + + "\001auth=Bearer " + password + "\001\001"))); + code = reply(); + if (code == 334) { + // The server's explanation of the failure; an empty line asks + // for the verdict. + command(""); + code = reply(); + } + } else if ("login".equalsIgnoreCase(chosen)) { + command("AUTH LOGIN"); + code = reply(); + if (code == 334) { + command(Base64.encode(WireBuffer.utf8(user))); + code = reply(); + } + if (code == 334) { + command(Base64.encode(WireBuffer.utf8(password))); + code = reply(); + } + } else if ("plain".equalsIgnoreCase(chosen)) { + command("AUTH PLAIN " + Base64.encode(WireBuffer.utf8("\000" + user + "\000" + + password))); + code = reply(); + } else { + throw new MailAuthenticationException("cn1.mail.smtp.auth=" + chosen + " is not a " + + "mechanism this client has; it has plain, login and xoauth2", 0); + } + if (code != 235) { + broken = true; + throw new MailAuthenticationException("The mail server refused the sign-in: " + + code + " " + firstLine(), code); + } + } + + private static boolean ascii(String value) { + for (int i = 0; i < value.length(); i++) { + if (value.charAt(i) > 126) { + return false; + } + } + return true; + } + + /// Sends one mail. When it returns the server has taken it for every + /// recipient. A [MailSendException] that is not permanent means nothing + /// was sent to anyone; one that is permanent means the server will not + /// take the mail, or took it for some recipients and refused the others + /// for good. + void send(String from, String[] recipients, byte[] mime) throws IOException { + if (maxSize > 0 && mime.length > maxSize) { + throw new MailSendException("The mail is " + mime.length + " bytes and the mail " + + "server accepts " + maxSize, 552, true); + } + boolean utf8 = !ascii(from); + for (String r : recipients) { + utf8 |= !ascii(r); + } + if (utf8 && !smtpUtf8) { + throw new MailSendException("An address of the mail is not ASCII and the mail " + + "server does not offer SMTPUTF8", 553, true); + } + StringBuilder mailFrom = new StringBuilder(64).append("MAIL FROM:<").append(from) + .append('>'); + if (sizeOffered) { + mailFrom.append(" SIZE=").append(mime.length); + } + if (utf8) { + mailFrom.append(" SMTPUTF8"); + } + int[] codes = new int[recipients.length]; + int mailCode; + int dataCode = 0; + String firstRefusal = null; + if (pipelining) { + stage(mailFrom.toString()); + for (String r : recipients) { + stage("RCPT TO:<" + r + ">"); + } + stage("DATA"); + flush(); + mailCode = reply(); + String mailText = firstLine(); + for (int i = 0; i < codes.length; i++) { + codes[i] = reply(); + if (firstRefusal == null && codes[i] / 100 != 2) { + firstRefusal = codes[i] + " " + firstLine(); + } + } + dataCode = reply(); + if (mailCode != 250) { + abandon(dataCode); + text.setLength(0); + text.append(mailText); + throw refused("the sender", mailCode); + } + } else { + command(mailFrom.toString()); + mailCode = reply(); + if (mailCode != 250) { + MailSendException err = refused("the sender", mailCode); + reset(); + throw err; + } + for (int i = 0; i < codes.length; i++) { + command("RCPT TO:<" + recipients[i] + ">"); + codes[i] = reply(); + if (firstRefusal == null && codes[i] / 100 != 2) { + firstRefusal = codes[i] + " " + firstLine(); + } + } + } + int accepted = 0; + int later = 0; + int refusedCode = 0; + for (int code : codes) { + if (code / 100 == 2) { + accepted++; + } else if (code / 100 == 4) { + later++; + refusedCode = code; + } else if (later == 0) { + refusedCode = code; + } + } + if (accepted == 0 || later > 0) { + // Nothing is sent to anyone: a recipient the server may take + // later means the whole mail is tried again, and sending it now + // to the others would send it to them twice. + abandon(dataCode); + throw new MailSendException("The mail server " + (later > 0 ? "deferred " : "refused ") + + (codes.length - accepted) + " of " + codes.length + " recipient(s): " + + firstRefusal, refusedCode, later == 0); + } + if (!pipelining) { + command("DATA"); + dataCode = reply(); + } + if (dataCode != 354) { + MailSendException err = refused("the message", dataCode); + if (!broken) { + reset(); + } + throw err; + } + body(mime); + int code = reply(); + if (code != 250) { + throw refused("the message", code); + } + lastUsed = System.currentTimeMillis(); + if (accepted < codes.length) { + throw new MailSendException("The mail was sent to " + accepted + " of " + + codes.length + " recipients; the mail server refused the rest: " + + firstRefusal, refusedCode, true); + } + } + + /// Gives up on a transaction. One that reached `DATA` and was told to go + /// ahead cannot be taken back with a command -- anything sent now is the + /// message -- so the connection is closed, which discards it. + private void abandon(int dataCode) throws IOException { + if (dataCode == 354) { + close(); + } else if (!broken) { + reset(); + } + } + + private void reset() throws IOException { + command("RSET"); + if (reply() != 250) { + broken = true; + } + } + + /// Writes the message and the line that ends it. A line that begins with + /// a dot is given a second one, which the server takes off again: alone + /// on a line, a dot is the end of the message. + private void body(byte[] mime) throws IOException { + int run = 0; + boolean lineStart = true; + for (int i = 0; i < mime.length; i++) { + byte b = mime[i]; + if (lineStart && b == '.') { + wire.writeBytes(mime, run, i - run); + wire.writeByte('.'); + run = i; + } + lineStart = b == '\n'; + if (lineStart && wire.pending() + (i + 1 - run) >= CHUNK) { + wire.writeBytes(mime, run, i + 1 - run); + run = i + 1; + flush(); + } + } + wire.writeBytes(mime, run, mime.length - run); + if (!lineStart) { + wire.writeByte('\r'); + wire.writeByte('\n'); + } + wire.writeByte('.'); + wire.writeByte('\r'); + wire.writeByte('\n'); + flush(); + } + + /// Whether the server still answers; a connection that sat idle may have + /// been closed from the other side. + boolean alive() { + if (broken) { + return false; + } + try { + command("NOOP"); + if (reply() != 250) { + broken = true; + } + } catch (IOException err) { + broken = true; + } + return !broken; + } + + /// Says goodbye when the session is in a state to, and closes the socket. + /// Never throws. + void close() { + if (!broken) { + broken = true; + try { + wire.writeBytes(WireBuffer.ascii("QUIT\r\n")); + wire.flush(); + } catch (IOException | RuntimeException err) { + // The socket is closed next either way. + } + } + tcp.close(); + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/smtp/SmtpTransport.java b/vm/backend/src/com/codename1/backend/mail/smtp/SmtpTransport.java new file mode 100644 index 00000000000..8625e8d1668 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/smtp/SmtpTransport.java @@ -0,0 +1,295 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.mail.smtp; + +import com.codename1.backend.Config; +import com.codename1.backend.mail.MailException; +import com.codename1.backend.mail.MailTransport; +import com.codename1.backend.mail.Mailers; +import java.io.IOException; +import java.util.ArrayList; +import java.util.List; +import java.util.concurrent.atomic.AtomicBoolean; + +/// Sends mail to an SMTP server: a provider's submission endpoint, a relay, +/// or a mail server of one's own. +/// +/// ```properties +/// cn1.mail.url=smtp://apikey:secret@smtp.example.com:587 +/// ``` +/// +/// `smtp://` connects in the clear and upgrades with STARTTLS before anything +/// else is said; a server that does not offer it is refused. `smtps://` +/// speaks TLS from the first byte, as port 465 does. The defaults are port +/// 587 and 465. A user name and password in the URL are percent-encoded where +/// they contain `@`, `:` or `/`. +/// +/// | Setting | Default | | +/// |---|---|---| +/// | `cn1.mail.username`, `cn1.mail.password` | | credentials, when the URL carries none | +/// | `cn1.mail.smtp.starttls` | `true` | `false` sends in the clear to a server with no STARTTLS -- a relay on localhost | +/// | `cn1.mail.smtp.auth` | | `plain`, `login` or `xoauth2`; unset takes what the server offers. With `xoauth2` the password is an OAuth 2 access token | +/// | `cn1.mail.smtp.helo` | `localhost` | the name the client gives for itself | +/// | `cn1.mail.smtp.ca` | | a PEM file of certificates to trust instead of the system's | +/// | `cn1.mail.timeout.connect` | 5000 | milliseconds a connection may take to open | +/// | `cn1.mail.timeout.read` | 60000 | milliseconds a reply may take | +/// | `cn1.mail.pool.max` | 2 | idle connections kept for the next mail | +/// | `cn1.mail.pool.idle` | 30000 | milliseconds an idle connection is kept | +/// +/// A password is never sent over a connection that is not encrypted, whatever +/// the settings say. +public final class SmtpTransport extends MailTransport { + private final String host; + private final int port; + private final boolean implicitTls; + private final boolean requireTls; + private final String user; + private final String password; + private final String mechanism; + private final String helo; + private final String caFile; + private final int connectTimeout; + private final int readTimeout; + private final int maxIdle; + private final long idleMillis; + private final List idle = new ArrayList(); + private final AtomicBoolean healthy = new AtomicBoolean(true); + private final AtomicBoolean closed = new AtomicBoolean(); + + /// Opens `smtp:` and `smtps:` URLs; what the generated wiring registers. + public static final class Factory implements Mailers.TransportFactory { + @Override + public MailTransport open(String url, Config config) throws IOException { + return new SmtpTransport(url, config); + } + } + + /// A transport for `url`, with the settings of `config`, which may be + /// null. Nothing connects until the first mail. + public SmtpTransport(String url, Config config) throws IOException { + String rest; + if (url.regionMatches(true, 0, "smtps://", 0, 8)) { + implicitTls = true; + rest = url.substring(8); + } else if (url.regionMatches(true, 0, "smtp://", 0, 7)) { + implicitTls = false; + rest = url.substring(7); + } else { + throw new IOException("Not an smtp:// or smtps:// URL"); + } + int cut = rest.indexOf('?'); + if (cut >= 0) { + rest = rest.substring(0, cut); + } + String u = null; + String p = null; + int at = rest.lastIndexOf('@'); + if (at >= 0) { + String credentials = rest.substring(0, at); + rest = rest.substring(at + 1); + int colon = credentials.indexOf(':'); + if (colon < 0) { + u = decode(credentials); + } else { + u = decode(credentials.substring(0, colon)); + p = decode(credentials.substring(colon + 1)); + } + } + int slash = rest.indexOf('/'); + if (slash >= 0) { + rest = rest.substring(0, slash); + } + int portValue = implicitTls ? 465 : 587; + int colon = rest.lastIndexOf(':'); + if (colon > 0 && rest.indexOf(']') < colon) { + try { + portValue = Integer.parseInt(rest.substring(colon + 1)); + } catch (NumberFormatException err) { + throw new IOException("The port of cn1.mail.url must be a number"); // NOPMD PreserveStackTrace - the cause adds nothing + } + rest = rest.substring(0, colon); + } + if (rest.startsWith("[") && rest.endsWith("]")) { + rest = rest.substring(1, rest.length() - 1); + } + host = rest.length() == 0 ? "localhost" : rest; + port = portValue; + if (u == null || u.length() == 0) { + u = setting(config, "cn1.mail.username", "spring.mail.username"); + p = setting(config, "cn1.mail.password", "spring.mail.password"); + } else if (p == null) { + p = setting(config, "cn1.mail.password", "spring.mail.password"); + } + user = u; + password = p; + mechanism = setting(config, "cn1.mail.smtp.auth", null); + String tls = setting(config, "cn1.mail.smtp.starttls", null); + requireTls = !"false".equalsIgnoreCase(tls); + String name = setting(config, "cn1.mail.smtp.helo", null); + helo = name == null ? "localhost" : name; + caFile = setting(config, "cn1.mail.smtp.ca", null); + connectTimeout = config == null ? 5000 : config.getInt("cn1.mail.timeout.connect", 5000); + readTimeout = config == null ? 60000 : config.getInt("cn1.mail.timeout.read", 60000); + maxIdle = Math.max(0, config == null ? 2 : config.getInt("cn1.mail.pool.max", 2)); + idleMillis = config == null ? 30000 : config.getInt("cn1.mail.pool.idle", 30000); + } + + private static String setting(Config config, String key, String fallback) throws IOException { + if (config == null) { + return null; + } + String v = config.get(key); + if ((v == null || v.trim().length() == 0) && fallback != null) { + v = config.get(fallback); + } + return v == null || v.trim().length() == 0 ? null : v.trim(); + } + + /// Percent-decodes a URL user or password. + private static String decode(String s) { + if (s.indexOf('%') < 0) { + return s; + } + byte[] out = new byte[s.length() * 3]; + int n = 0; + for (int i = 0; i < s.length(); i++) { + char c = s.charAt(i); + if (c == '%' && i + 2 < s.length() && hex(s.charAt(i + 1)) >= 0 + && hex(s.charAt(i + 2)) >= 0) { + out[n++] = (byte) (hex(s.charAt(i + 1)) * 16 + hex(s.charAt(i + 2))); + i += 2; + } else { + byte[] one = com.codename1.impl.backend.WireBuffer.utf8(String.valueOf(c)); + System.arraycopy(one, 0, out, n, one.length); + n += one.length; + } + } + byte[] exact = new byte[n]; + System.arraycopy(out, 0, exact, 0, n); + return com.codename1.impl.backend.WireBuffer.fromUtf8(exact); + } + + private static int hex(char c) { + if (c >= '0' && c <= '9') { + return c - '0'; + } + if (c >= 'a' && c <= 'f') { + return c - 'a' + 10; + } + if (c >= 'A' && c <= 'F') { + return c - 'A' + 10; + } + return -1; + } + + @Override + public String getName() { + return implicitTls ? "smtps" : "smtp"; + } + + /// The server's host name. + public String getHost() { + return host; + } + + /// The server's port. + public int getPort() { + return port; + } + + private SmtpConnection connect() throws IOException { + return SmtpConnection.open(host, port, implicitTls, requireTls, caFile, helo, user, + password, mechanism, connectTimeout, readTimeout); + } + + /// An idle connection the server still answers on, or a new one. + private SmtpConnection borrow() throws IOException { + while (true) { + SmtpConnection c; // NOPMD CloseResource - returned to the caller or closed below + synchronized (idle) { + if (idle.isEmpty()) { + return connect(); + } + c = idle.remove(idle.size() - 1); + } + long quiet = System.currentTimeMillis() - c.lastUsed(); + if (quiet <= idleMillis && (quiet <= 1000 || c.alive())) { + return c; + } + c.close(); + } + } + + private void release(SmtpConnection c) { + if (!c.isBroken()) { + synchronized (idle) { + // Read under the lock close() empties the list under, or a + // send that ends as the transport closes leaves a connection + // in a list nobody reads again. + if (!closed.get() && idle.size() < maxIdle) { + idle.add(c); + return; + } + } + } + c.close(); + } + + @Override + public void send(String from, String[] recipients, byte[] mime) throws IOException { + if (closed.get()) { + throw new MailException("The mail transport is closed"); + } + SmtpConnection c; // NOPMD CloseResource - released below + try { + c = borrow(); + } catch (IOException err) { + healthy.set(false); + throw err; + } + healthy.set(true); + try { + c.send(from, recipients, mime); + } finally { + release(c); + } + } + + @Override + public boolean isHealthy() { + return healthy.get(); + } + + @Override + public void close() { + closed.set(true); + List all; + synchronized (idle) { + all = new ArrayList(idle); + idle.clear(); + } + for (SmtpConnection c : all) { + c.close(); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/mail/smtp/package-info.java b/vm/backend/src/com/codename1/backend/mail/smtp/package-info.java new file mode 100644 index 00000000000..da8ae91bed2 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/mail/smtp/package-info.java @@ -0,0 +1,26 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// The SMTP transport: [com.codename1.backend.mail.smtp.SmtpTransport] sends +/// through a provider's submission endpoint or a mail server of one's own, +/// with STARTTLS or implicit TLS, sign-in and a small pool of connections. +package com.codename1.backend.mail.smtp; diff --git a/vm/backend/src/com/codename1/backend/messaging/OutboxTransport.java b/vm/backend/src/com/codename1/backend/messaging/OutboxTransport.java index c8c9f0c449b..6060266779b 100644 --- a/vm/backend/src/com/codename1/backend/messaging/OutboxTransport.java +++ b/vm/backend/src/com/codename1/backend/messaging/OutboxTransport.java @@ -67,6 +67,18 @@ final class OutboxTransport extends MessageTransport { this.broker = broker; this.db = db; this.sweepAfter = Math.max(100, sweepAfterMillis); + // The table is made here, not by the first send. MySQL and MariaDB end + // the open transaction on any DDL, a CREATE TABLE IF NOT EXISTS of a + // table that exists included: a first send inside a transaction then + // wrote its row outside it, and a message the caller rolled back was + // relayed by the sweep. A database that is down now is tried again by + // the first send. + try { + ensureTable(); + } catch (IOException | RuntimeException err) { + System.err.println("cn1: the messaging outbox table could not be created yet: " + + err.getMessage()); + } } @Override diff --git a/vm/backend/src/com/codename1/backend/security/DefaultLoginPageGeneratingFilter.java b/vm/backend/src/com/codename1/backend/security/DefaultLoginPageGeneratingFilter.java index 785d26ec6f1..a3922b52466 100644 --- a/vm/backend/src/com/codename1/backend/security/DefaultLoginPageGeneratingFilter.java +++ b/vm/backend/src/com/codename1/backend/security/DefaultLoginPageGeneratingFilter.java @@ -33,12 +33,15 @@ public final class DefaultLoginPageGeneratingFilter implements SecurityFilter { private final String passwordParameter; private final String rememberMeParameter; private final java.util.List links; + private final java.util.List resetPages; DefaultLoginPageGeneratingFilter(String loginPage, String processingUrl, String usernameParameter, String passwordParameter, String rememberMeParameter, - java.util.List links) { + java.util.List links, + java.util.List resetPages) { this.links = links; + this.resetPages = resetPages; this.rememberMeParameter = rememberMeParameter; this.loginPage = loginPage; this.processingUrl = processingUrl; @@ -66,6 +69,11 @@ public HttpServer.Response doFilter(HttpServer.Request request, FilterChain chai if (error) { page.append("

Bad credentials

\n"); } + // Only a chain that mails resets sends anyone here with this. + if (!resetPages.isEmpty() && request.queryParam("reset") != null) { + page.append("

Your password was changed. Sign in with the new " + + "one.

\n"); + } if (loggedOut) { page.append("

You have been signed out

\n"); } @@ -88,6 +96,12 @@ public HttpServer.Response doFilter(HttpServer.Request request, FilterChain chai .append(Responses.escape(token.getToken())).append("\">\n"); } page.append("\n\n"); + // Offered only by a chain that mails resets, which is one that has a + // way of sending mail. + if (!resetPages.isEmpty()) { + page.append("

Forgot your password?

\n"); + } // The other ways of signing in the chain declares: a link to each. for (String[] link : links) { page.append("

Sign in with ") diff --git a/vm/backend/src/com/codename1/backend/security/EmailVerificationConfigurer.java b/vm/backend/src/com/codename1/backend/security/EmailVerificationConfigurer.java new file mode 100644 index 00000000000..3e72c718515 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/EmailVerificationConfigurer.java @@ -0,0 +1,109 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security; + +import com.codename1.backend.security.mail.EmailVerificationService; + +/// The address a mailed verification link opens. +/// +/// ```java +/// http.emailVerification(Customizer.withDefaults()); +/// ``` +/// +/// `GET /verify-email?token=...` is a page with one button, and pressing it -- +/// a `POST` -- records the address as proved and sends the user to +/// `/?verified`; a link that is no longer good is answered 400. Opening the +/// link proves nothing by itself, because a mail gateway opens every link in a +/// mail before its reader does. The address is open to everyone: a link is +/// often opened on another device than the one that is signed in. +/// +/// The application mails the link itself, where it takes the address, with +/// [#linkUrl] as where it leads: +/// +/// ```java +/// verification.send(username, email, "https://example.com/verify-email"); +/// ``` +/// +/// The work is an [EmailVerificationService]'s: the one given here, or the +/// application's bean of that type. +public final class EmailVerificationConfigurer extends SecurityConfigurer { + private EmailVerificationService service; + private String confirmUrl = "/verify-email"; + private String successUrl = "/?verified"; + + EmailVerificationConfigurer() { + } + + /// What checks the link, in place of the application's bean. + public EmailVerificationConfigurer service(EmailVerificationService service) { + this.service = service; + return this; + } + + /// The path the link opens; `/verify-email` unless set. + public EmailVerificationConfigurer confirmUrl(String confirmUrl) { + this.confirmUrl = Responses.path(confirmUrl, "confirmUrl"); + return this; + } + + /// Where the user is sent once the address is proved; `/?verified` unless + /// set. + public EmailVerificationConfigurer successUrl(String successUrl) { + this.successUrl = Responses.path(successUrl, "successUrl"); + return this; + } + + /// The whole address to give + /// [EmailVerificationService#send(String, String, String)]: the server's + /// address from `cn1.security.base-url` and the path the link opens. + /// + /// #### Throws + /// + /// - `IllegalStateException`: the setting is missing + public String linkUrl() { + String base = PasswordResetConfigurer.base(getBuilder(), null); + if (base == null) { + throw new IllegalStateException("A link cannot be mailed without knowing the " + + "address this server is reached at. Set " + + PasswordResetConfigurer.BASE_URL + "=https://your.host in " + + "application.properties."); + } + return base + confirmUrl; + } + + @Override + public void init(HttpSecurity http) { + EmailVerificationService verification = service != null ? service + : http.getSharedObject(EmailVerificationService.class); + if (verification == null) { + throw new IllegalStateException("emailVerification() needs an " + + "EmailVerificationService, and this application has none. Declare a bean " + + "of that type, or call service(...) on the emailVerification() " + + "configurer."); + } + http.addFilter(new EmailVerificationFilter(verification, confirmUrl, successUrl), + HttpSecurity.ORDER_EMAIL_VERIFICATION); + http.permit(AntPathRequestMatcher.antMatcher("GET", confirmUrl)); + http.permit(AntPathRequestMatcher.antMatcher("POST", confirmUrl)); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/EmailVerificationFilter.java b/vm/backend/src/com/codename1/backend/security/EmailVerificationFilter.java new file mode 100644 index 00000000000..9d661bbccfc --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/EmailVerificationFilter.java @@ -0,0 +1,90 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security; + +import com.codename1.backend.HttpServer; +import com.codename1.backend.security.mail.EmailVerificationService; + +/// Takes a mailed verification link; see [EmailVerificationConfigurer]. +public final class EmailVerificationFilter implements SecurityFilter { + private final EmailVerificationService service; + private final String confirmUrl; + private final String successUrl; + + EmailVerificationFilter(EmailVerificationService service, String confirmUrl, + String successUrl) { + this.service = service; + this.confirmUrl = confirmUrl; + this.successUrl = successUrl; + } + + @Override + public HttpServer.Response doFilter(HttpServer.Request request, FilterChain chain) + throws Exception { + boolean get = "GET".equals(request.getMethod()); + if (!(get || "POST".equals(request.getMethod())) + || !confirmUrl.equals(SecurityExchange.path(request))) { + return chain.doFilter(request); + } + String token = get ? request.queryParam("token") : Responses.param(request, "token"); + if (token == null || !PasswordResetFilter.isToken(token)) { + return gone(); + } + if (get) { + // Opening the link proves nothing: a mail gateway opens every link + // in a mail before its reader sees it, and would prove an address + // for whoever typed it. The page asks, and the answer is a POST, + // which no scanner sends. + StringBuilder html = new StringBuilder(768); + html.append("\n\n\n\n" + + "\n" + + "Confirm your email address\n\n\n" + + "

\n

Confirm your email address

\n" + + "\n"); + CsrfToken csrf = CsrfFilter.getToken(request); + if (csrf != null) { + html.append("\n"); + } + html.append("\n\n\n" + + "\n"); + // The address of this page carries the token. + return Responses.html(200, html.toString()).header("Cache-Control", "no-store") + .header("Referrer-Policy", "no-referrer"); + } + if (service.confirm(token) == null) { + return gone(); + } + return Responses.redirect(successUrl).header("Referrer-Policy", "no-referrer"); + } + + private static HttpServer.Response gone() { + return Responses.html(400, "\n\n\n" + + "\nLink no longer good\n\n" + + "\n

That link is no longer good. Ask for another" + + "

\n\n\n").header("Cache-Control", "no-store"); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/FormLoginConfigurer.java b/vm/backend/src/com/codename1/backend/security/FormLoginConfigurer.java index 364bffc7915..d6227131554 100644 --- a/vm/backend/src/com/codename1/backend/security/FormLoginConfigurer.java +++ b/vm/backend/src/com/codename1/backend/security/FormLoginConfigurer.java @@ -170,7 +170,7 @@ public void configure(HttpSecurity http) { if (!customLoginPage) { http.addFilter(new DefaultLoginPageGeneratingFilter(loginPage, processingUrl(), usernameParameter, passwordParameter, http.rememberMeParameter(), - http.loginLinks()), + http.loginLinks(), http.passwordResetPages()), HttpSecurity.ORDER_LOGIN_PAGE); } } diff --git a/vm/backend/src/com/codename1/backend/security/HttpSecurity.java b/vm/backend/src/com/codename1/backend/security/HttpSecurity.java index b4030f2a0ce..63569fe0423 100644 --- a/vm/backend/src/com/codename1/backend/security/HttpSecurity.java +++ b/vm/backend/src/com/codename1/backend/security/HttpSecurity.java @@ -114,6 +114,8 @@ public final class HttpSecurity { static final int ORDER_WEBAUTHN_LOGIN = 950; static final int ORDER_FORM_LOGIN = 1000; static final int ORDER_SECOND_FACTOR = 1050; + static final int ORDER_PASSWORD_RESET = 1060; + static final int ORDER_EMAIL_VERIFICATION = 1070; static final int ORDER_LOGIN_PAGE = 1100; static final int ORDER_API_KEY = 1200; static final int ORDER_BEARER_TOKEN = 1300; @@ -179,6 +181,10 @@ interface RequestCacheSource { Integer.valueOf(ORDER_FORM_LOGIN)); filterOrder.put(PACKAGE + "SecondFactorAuthenticationFilter", Integer.valueOf(ORDER_SECOND_FACTOR)); + filterOrder.put(PACKAGE + "PasswordResetFilter", + Integer.valueOf(ORDER_PASSWORD_RESET)); + filterOrder.put(PACKAGE + "EmailVerificationFilter", + Integer.valueOf(ORDER_EMAIL_VERIFICATION)); filterOrder.put(PACKAGE + "DefaultLoginPageGeneratingFilter", Integer.valueOf(ORDER_LOGIN_PAGE)); filterOrder.put(PACKAGE + "ApiKeyAuthenticationFilter", Integer.valueOf(ORDER_API_KEY)); @@ -337,6 +343,31 @@ public HttpSecurity mfa(Customizer customizer) { return this; } + /// A password reset by mail: the page that asks for one and the page the + /// mailed link opens; see [PasswordResetConfigurer]. + public HttpSecurity passwordReset(Customizer customizer) { + PasswordResetConfigurer configurer = getConfigurer(PasswordResetConfigurer.class); + if (configurer == null) { + configurer = new PasswordResetConfigurer(); + apply(configurer); + } + customizer.customize(configurer); + return this; + } + + /// The address a mailed verification link opens; see + /// [EmailVerificationConfigurer]. + public HttpSecurity emailVerification(Customizer customizer) { + EmailVerificationConfigurer configurer = + getConfigurer(EmailVerificationConfigurer.class); + if (configurer == null) { + configurer = new EmailVerificationConfigurer(); + apply(configurer); + } + customizer.customize(configurer); + return this; + } + /// Passkeys: registering one for a user who is signed in, and signing in /// with one; see [WebAuthnConfigurer]. Brings sign-out with it, as /// [#formLogin] does. @@ -860,6 +891,22 @@ List loginLinks() { return loginLinks; } + private final List passwordResetPages = new ArrayList(); + + /// Says that this chain mails password resets, and where one is asked + /// for: a generated login page then links there. Told from a configurer's + /// init(). + void passwordResetPage(String path) { + passwordResetPages.add(path); + } + + /// The page a reset is asked for at, in a list that is empty when the + /// chain mails none. The list itself is handed out, not a copy: the login + /// page may be built before the part that fills it has run. + List passwordResetPages() { + return passwordResetPages; + } + /// Says that a part of this chain serves the login page; from a /// configurer's init(). Another part that would serve one then does not. void servesLoginPage() { diff --git a/vm/backend/src/com/codename1/backend/security/MfaConfigurer.java b/vm/backend/src/com/codename1/backend/security/MfaConfigurer.java index 9db43b67bf0..25a87ed87a6 100644 --- a/vm/backend/src/com/codename1/backend/security/MfaConfigurer.java +++ b/vm/backend/src/com/codename1/backend/security/MfaConfigurer.java @@ -22,6 +22,7 @@ */ package com.codename1.backend.security; +import com.codename1.backend.security.mfa.EmailCodeService; import com.codename1.backend.security.mfa.RecoveryCodeService; import com.codename1.backend.security.mfa.TotpService; import com.codename1.backend.security.ratelimit.InMemoryRateLimiter; @@ -50,6 +51,11 @@ /// of that type, and recovery codes are accepted when there is a /// [RecoveryCodeService] the same way. /// +/// An [EmailCodeService], given here or declared as a bean, adds a code sent +/// by mail: to a user it is enabled for and who has no authenticator app, as +/// their password is accepted; to one who has both, when they ask for it on +/// the page. Either service is enough for `mfa()`; with neither it is refused. +/// /// #### How many guesses, and who can be kept out /// /// Wrong codes are counted, and too many are answered 429. Three numbers @@ -168,6 +174,7 @@ public final class MfaConfigurer extends SecurityConfigurer { public static final String ATTEMPTS_WINDOW = "cn1.security.mfa.attemptsWindowSeconds"; private TotpService totp; + private EmailCodeService emailCodes; private RecoveryCodeService recoveryCodes; private String page = "/login/mfa"; private boolean customPage; @@ -191,6 +198,13 @@ public MfaConfigurer totpService(TotpService totpService) { return this; } + /// What mails and checks sign-in codes, in place of the application's + /// bean. + public MfaConfigurer emailCodeService(EmailCodeService emailCodeService) { + this.emailCodes = emailCodeService; + return this; + } + /// What checks recovery codes, in place of the application's bean. public MfaConfigurer recoveryCodeService(RecoveryCodeService recoveryCodeService) { this.recoveryCodes = recoveryCodeService; @@ -337,10 +351,13 @@ private String processing() { @Override public void init(HttpSecurity http) { TotpService codes = totp != null ? totp : http.getSharedObject(TotpService.class); - if (codes == null) { + EmailCodeService mailed = emailCodes != null ? emailCodes + : http.getSharedObject(EmailCodeService.class); + if (codes == null && mailed == null) { throw new IllegalStateException("mfa() needs something to check codes with, and " - + "this application has none. Declare a TotpService bean, or call " - + "totpService(...) on the mfa() configurer."); + + "this application has none. Declare a TotpService or an " + + "EmailCodeService bean, or call totpService(...) or " + + "emailCodeService(...) on the mfa() configurer."); } RecoveryCodeService recovery = recoveryCodes != null ? recoveryCodes : http.getSharedObject(RecoveryCodeService.class); @@ -353,7 +370,7 @@ public void init(HttpSecurity http) { success = saved; } RateLimiter[] counting = attempts(http); - filter = new SecondFactorAuthenticationFilter(codes, recovery, page, !customPage, + filter = new SecondFactorAuthenticationFilter(codes, mailed, recovery, page, !customPage, processing(), codeParameter, pendingSeconds * 1000L, counting[0], counting[1], success, new SimpleUrlAuthenticationFailureHandler(http.loginPage() + "?error"), clock); diff --git a/vm/backend/src/com/codename1/backend/security/PasswordResetConfigurer.java b/vm/backend/src/com/codename1/backend/security/PasswordResetConfigurer.java new file mode 100644 index 00000000000..269c19c89f2 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/PasswordResetConfigurer.java @@ -0,0 +1,187 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security; + +import com.codename1.backend.security.mail.PasswordResetService; +import com.codename1.backend.security.ratelimit.InMemoryRateLimiter; +import com.codename1.backend.security.ratelimit.RateLimiter; + +/// The pages of a password reset by mail. +/// +/// ```java +/// http.formLogin(Customizer.withDefaults()) +/// .passwordReset(Customizer.withDefaults()); +/// ``` +/// +/// ```properties +/// cn1.security.base-url=https://example.com +/// ``` +/// +/// `GET /forgot-password` asks for a user name and `POST /forgot-password` +/// mails that user a link, answering the same whether or not there is such a +/// user. The link opens `GET /reset-password`, which asks for the new +/// password, and `POST /reset-password` sets it and sends the user to the +/// sign-in page. All four are open to everyone, whatever the authorization +/// rules say. +/// +/// The work is a [PasswordResetService]'s: the one given here, or the +/// application's bean of that type. +/// +/// #### The address in the link +/// +/// The link is mailed to somebody other than whoever asked for it, so it is +/// never built from the request's `Host` header, which the caller chooses. +/// It starts with `cn1.security.base-url`, or what [#baseUrl] is given. With +/// neither, a request to `localhost` gets a link to itself, which is what a +/// developer's machine needs, and any other request is refused with an error +/// that names the setting. +/// +/// #### How often +/// +/// Asking is counted for the client's network and for the name asked about, +/// 5 in 15 minutes each unless [#attemptLimiter] says otherwise, and more are +/// answered 429. The count for a name is kept whether or not there is such a +/// user, so it tells nobody which there are; it is what stops one person's +/// mailbox being filled from many addresses. +public final class PasswordResetConfigurer extends SecurityConfigurer { + /// The setting that holds the address the server is reached at from + /// outside, `https://example.com`, with no path. + public static final String BASE_URL = "cn1.security.base-url"; + + private PasswordResetService service; + private String requestUrl = "/forgot-password"; + private String resetUrl = "/reset-password"; + private String baseUrl; + private boolean servePages = true; + private RateLimiter attempts; + private boolean attemptsGiven; + + PasswordResetConfigurer() { + } + + /// What mails the link and sets the password, in place of the + /// application's bean. + public PasswordResetConfigurer service(PasswordResetService service) { + this.service = service; + return this; + } + + /// Where a reset is asked for; `/forgot-password` unless set. + public PasswordResetConfigurer requestUrl(String requestUrl) { + this.requestUrl = Responses.path(requestUrl, "requestUrl"); + return this; + } + + /// Where the mailed link leads and the new password is posted; + /// `/reset-password` unless set. + public PasswordResetConfigurer resetUrl(String resetUrl) { + this.resetUrl = Responses.path(resetUrl, "resetUrl"); + return this; + } + + /// The address the server is reached at from outside, in place of the + /// `cn1.security.base-url` setting. + public PasswordResetConfigurer baseUrl(String baseUrl) { + this.baseUrl = checkBase(baseUrl, "baseUrl"); + return this; + } + + /// False leaves the two `GET` pages to the application, which then posts + /// the fields `username`, and `token`, `password` and `confirm`, to the + /// same paths. + public PasswordResetConfigurer servePages(boolean servePages) { + this.servePages = servePages; + return this; + } + + /// What counts requests for a link, in place of the application's + /// [RateLimiter] bean and of the count kept in this process; null to + /// count none. + public PasswordResetConfigurer attemptLimiter(RateLimiter attemptLimiter) { + this.attempts = attemptLimiter; + this.attemptsGiven = true; + return this; + } + + static String checkBase(String value, String what) { + if (value == null) { + return null; + } + String url = value.trim(); + while (url.endsWith("/")) { + url = url.substring(0, url.length() - 1); + } + boolean scheme = url.regionMatches(true, 0, "https://", 0, 8) + || url.regionMatches(true, 0, "http://", 0, 7); + if (!scheme || url.indexOf('/', url.indexOf("//") + 2) >= 0 || url.indexOf('?') >= 0 + || url.indexOf('#') >= 0) { + throw new IllegalArgumentException(what + " must be the server's address and " + + "nothing after it, such as https://example.com: " + value); + } + return url; + } + + /// The configured address, or null when requests to localhost alone are + /// served. + static String base(HttpSecurity http, String given) { + if (given != null) { + return given; + } + try { + String set = http.getConfig() == null ? null : http.getConfig().get(BASE_URL, null); + return set == null || set.trim().length() == 0 ? null : checkBase(set, BASE_URL); + } catch (java.io.IOException err) { + throw new IllegalStateException(BASE_URL + " could not be read from the " + + "configuration: " + err.getMessage(), err); + } + } + + @Override + public void init(HttpSecurity http) { + PasswordResetService reset = service != null ? service + : http.getSharedObject(PasswordResetService.class); + if (reset == null) { + throw new IllegalStateException("passwordReset() needs a PasswordResetService, and " + + "this application has none. Declare a bean of that type, or call " + + "service(...) on the passwordReset() configurer."); + } + RateLimiter counting = attempts; + if (!attemptsGiven) { + RateLimiter bean = http.uniqueSharedObject(RateLimiter.class, + "The password reset's limit", "passwordReset().attemptLimiter(...)"); + RateLimiter derived = bean == null ? null : bean.derive("password-reset", 5, 900); + counting = derived != null ? derived : bean != null ? bean + : new InMemoryRateLimiter(5, 900); + } + http.addFilter(new PasswordResetFilter(reset, requestUrl, resetUrl, + base(http, baseUrl), servePages, counting, http.loginPage()), + HttpSecurity.ORDER_PASSWORD_RESET); + // A generated login page offers the way here, and says so when a + // reset sends the user back to it. + http.passwordResetPage(requestUrl); + http.permit(AntPathRequestMatcher.antMatcher("GET", requestUrl)); + http.permit(AntPathRequestMatcher.antMatcher("POST", requestUrl)); + http.permit(AntPathRequestMatcher.antMatcher("GET", resetUrl)); + http.permit(AntPathRequestMatcher.antMatcher("POST", resetUrl)); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/PasswordResetFilter.java b/vm/backend/src/com/codename1/backend/security/PasswordResetFilter.java new file mode 100644 index 00000000000..b817a7e90af --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/PasswordResetFilter.java @@ -0,0 +1,227 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security; + +import com.codename1.backend.HttpServer; +import com.codename1.backend.security.mail.PasswordResetService; +import com.codename1.backend.security.ratelimit.RateLimitKeyResolver; +import com.codename1.backend.security.ratelimit.RateLimitKeys; +import com.codename1.backend.security.ratelimit.RateLimiter; + +/// Serves the pages of a password reset and takes what they post; see +/// [PasswordResetConfigurer]. +public final class PasswordResetFilter implements SecurityFilter { + private final PasswordResetService service; + private final String requestUrl; + private final String resetUrl; + private final String baseUrl; + private final boolean servePages; + private final RateLimiter attempts; + private final String loginPage; + private final RateLimitKeyResolver network = RateLimitKeys.clientNetwork(); + + PasswordResetFilter(PasswordResetService service, String requestUrl, String resetUrl, + String baseUrl, boolean servePages, RateLimiter attempts, String loginPage) { + this.service = service; + this.requestUrl = requestUrl; + this.resetUrl = resetUrl; + this.baseUrl = baseUrl; + this.servePages = servePages; + this.attempts = attempts; + this.loginPage = loginPage; + } + + @Override + public HttpServer.Response doFilter(HttpServer.Request request, FilterChain chain) + throws Exception { + String path = SecurityExchange.path(request); + boolean get = "GET".equals(request.getMethod()); + boolean post = "POST".equals(request.getMethod()); + if (requestUrl.equals(path)) { + if (post) { + return ask(request); + } + if (get && servePages) { + return askPage(request); + } + } else if (resetUrl.equals(path)) { + if (post) { + return set(request); + } + if (get && servePages) { + return setPage(request); + } + } + return chain.doFilter(request); + } + + /// The address a link to this server starts with. The request's own host + /// is believed only when it is this machine: a link is mailed to somebody + /// who did not send the request. + static String origin(HttpServer.Request request, String configured) { + if (configured != null) { + return configured; + } + String host = request.getHeader("Host"); + String name = host == null ? "" : host; + int port = name.lastIndexOf(':'); + if (port > name.lastIndexOf(']')) { + name = name.substring(0, port); + } + boolean digits = host != null; + for (int iter = port + 1 ; digits && port >= 0 && iter < host.length() ; iter++) { + digits = host.charAt(iter) >= '0' && host.charAt(iter) <= '9'; + } + if (!digits || !("localhost".equalsIgnoreCase(name) || "127.0.0.1".equals(name) // NOPMD AvoidUsingHardCodedIP - loopback is the one host trusted without a setting + || "[::1]".equals(name))) { + throw new IllegalStateException("A link cannot be mailed without knowing the " + + "address this server is reached at. Set " + + PasswordResetConfigurer.BASE_URL + "=https://your.host in " + + "application.properties; only a request to localhost is served without " + + "it."); + } + return (request.isSecure() ? "https://" : "http://") + host; + } + + private HttpServer.Response ask(HttpServer.Request request) { + String username = Responses.param(request, "username"); + String origin = origin(request, baseUrl); + if (attempts != null) { + String from = network.resolve(request); + String here = "password-reset:" + (from == null ? "net:none" : from); + if (!attempts.tryAcquire(here)) { + return tooMany(attempts.retryAfterSeconds(here)); + } + // Per account as well, or one caller spread over many addresses + // fills a mailbox. It lets anybody who knows a user name use up + // that user's allowance for a quarter of an hour; a reset is + // not how a user signs in, and mail to them without end is + // the worse of the two. + if (username != null && username.length() > 0 && username.length() <= 320) { + String who = "password-reset:user:" + SecuritySchema.usernameKey(username); + if (!attempts.tryAcquire(who)) { + return tooMany(attempts.retryAfterSeconds(who)); + } + } + } + service.request(username, origin + resetUrl); + return Responses.redirect(requestUrl + "?sent"); + } + + private HttpServer.Response set(HttpServer.Request request) { + String token = Responses.param(request, "token"); + String password = Responses.param(request, "password"); + String confirm = Responses.param(request, "confirm"); + if (token == null || !isToken(token)) { + return Responses.redirect(requestUrl + "?expired"); + } + // Looked at before the token is, so that a slip of the keyboard does + // not cost the user their link. + if (password == null || (confirm != null && !confirm.equals(password)) + || !service.isAcceptable(password)) { + return Responses.redirect(resetUrl + "?token=" + token + "&invalid"); + } + if (!service.reset(token, password)) { + return Responses.redirect(requestUrl + "?expired"); + } + return Responses.redirect(loginPage + "?reset"); + } + + /// Whether `token` is written as a token is, and so can go into a URL as + /// it stands. + static boolean isToken(String token) { + if (token.length() != 43) { + return false; + } + for (int iter = 0 ; iter < token.length() ; iter++) { + char c = token.charAt(iter); + if (!((c >= 'a' && c <= 'z') || (c >= 'A' && c <= 'Z') || (c >= '0' && c <= '9') + || c == '-' || c == '_')) { + return false; + } + } + return true; + } + + private static HttpServer.Response tooMany(long wait) { + return Responses.status(429, "Too Many Requests") + .header("Retry-After", String.valueOf(wait < 1 ? 1 : wait)); + } + + private static StringBuilder open(String title, String action) { + StringBuilder html = new StringBuilder(1024); + html.append("\n\n\n\n"); + html.append("\n"); + html.append("").append(title).append("\n\n\n"); + html.append("
\n

").append(title).append("

\n"); + return html; + } + + private static HttpServer.Response close(HttpServer.Request request, StringBuilder html) { + CsrfToken token = CsrfFilter.getToken(request); + if (token != null) { + html.append("\n"); + } + html.append("\n
\n\n\n"); + // The address of the second page carries the token: it is not to be + // kept by a cache or handed on to whatever the page links to. + return Responses.html(200, html.toString()).header("Cache-Control", "no-store") + .header("Referrer-Policy", "no-referrer"); + } + + private HttpServer.Response askPage(HttpServer.Request request) { + StringBuilder html = open("Reset your password", requestUrl); + if (request.queryParam("sent") != null) { + html.append("

If there is an account by that name, a link to " + + "reset its password is on its way to its address

\n"); + } else if (request.queryParam("expired") != null) { + html.append("

That link is no longer good. Ask for another

\n"); + } + html.append("

\n

\n"); + return close(request, html); + } + + private HttpServer.Response setPage(HttpServer.Request request) { + String token = request.queryParam("token"); + if (token == null || !isToken(token)) { + return Responses.redirect(requestUrl + "?expired"); + } + StringBuilder html = open("Choose a new password", resetUrl); + if (request.queryParam("invalid") != null) { + html.append("

The two passwords must be the same, and at least ") + .append(service.getMinimumLength()).append(" characters long

\n"); + } + html.append("\n

\n

\n

\n

\n"); + return close(request, html); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/SecondFactorAuthenticationFilter.java b/vm/backend/src/com/codename1/backend/security/SecondFactorAuthenticationFilter.java index 7ca28f2e66d..d29bd1a8dfa 100644 --- a/vm/backend/src/com/codename1/backend/security/SecondFactorAuthenticationFilter.java +++ b/vm/backend/src/com/codename1/backend/security/SecondFactorAuthenticationFilter.java @@ -25,6 +25,7 @@ import com.codename1.backend.HttpServer; import com.codename1.backend.HttpSession; import com.codename1.backend.security.core.userdetails.User; +import com.codename1.backend.security.mfa.EmailCodeService; import com.codename1.backend.security.mfa.RecoveryCodeService; import com.codename1.backend.security.mfa.TotpService; import com.codename1.backend.security.ratelimit.RateLimitKeyResolver; @@ -46,7 +47,10 @@ /// /// It is also the filter that takes the code, at `POST /login/mfa` unless /// changed, in the field `code`: a one-time code from the user's authenticator -/// app, or one of their recovery codes. A right one completes the sign-in +/// app, the code that was mailed to them, or one of their recovery codes. A +/// user whose only second factor is the mailed code is sent one as they reach +/// the page; one who has an authenticator app too asks for it there, by +/// posting the field `send`. A right code completes the sign-in /// through the chain's [SessionSignIn] -- the session id changes, the context /// is stored, remember-me is issued if it was asked for at the first step, and /// the user goes where they were going. @@ -60,6 +64,7 @@ public final class SecondFactorAuthenticationFilter implements SecurityFilter, S public static final String PENDING = "CN1_SECURITY_SECOND_FACTOR_PENDING"; private final TotpService totp; + private final EmailCodeService emailCodes; private final RecoveryCodeService recoveryCodes; private final String page; private final boolean servePage; @@ -78,12 +83,14 @@ public final class SecondFactorAuthenticationFilter implements SecurityFilter, S private SessionSignIn signIn; private HttpSessionSecurityContextRepository kinds; - SecondFactorAuthenticationFilter(TotpService totp, RecoveryCodeService recoveryCodes, + SecondFactorAuthenticationFilter(TotpService totp, EmailCodeService emailCodes, + RecoveryCodeService recoveryCodes, String page, boolean servePage, String processingUrl, String codeParameter, long pendingMillis, RateLimiter attempts, RateLimiter addressAttempts, AuthenticationSuccessHandler successHandler, AuthenticationFailureHandler expiredHandler, Clock clock) { this.totp = totp; + this.emailCodes = emailCodes; this.recoveryCodes = recoveryCodes; this.page = page; this.servePage = servePage; @@ -110,12 +117,25 @@ void kinds(HttpSessionSecurityContextRepository repository) { this.kinds = repository; } + private boolean app(String name) { + return totp != null && totp.isEnabled(name); + } + + private boolean mailed(String name) { + return emailCodes != null && emailCodes.isEnabled(name); + } + // ------------------------------------------------------------ the policy @Override public HttpServer.Response intercept(HttpServer.Request request, Authentication authentication, boolean rememberMe) { - if (authentication == null || !totp.isEnabled(authentication.getName())) { + if (authentication == null) { + return null; + } + boolean app = app(authentication.getName()); + boolean mailed = mailed(authentication.getName()); + if (!app && !mailed) { return null; } Map pending = new LinkedHashMap(); @@ -138,12 +158,18 @@ public HttpServer.Response intercept(HttpServer.Request request, session.removeAttribute(HttpSessionSecurityContextRepository.SPRING_SECURITY_CONTEXT_KEY); session.changeSessionId(); session.setAttribute(PENDING, pending); + if (mailed && !app) { + // The only way in, so it is not waited to be asked for. A user who + // has an app as well is not mailed at every sign-in: they ask. + emailCodes.send(authentication.getName()); + } return Responses.redirect(page); } @Override public boolean requires(Authentication authentication) { - return authentication != null && totp.isEnabled(authentication.getName()); + return authentication != null && (app(authentication.getName()) + || mailed(authentication.getName())); } /// A passkey that verified the user signed them in. Whoever ran the count @@ -151,7 +177,7 @@ public boolean requires(Authentication authentication) { /// so their next one-time code is not refused for somebody else's guesses. @Override public void satisfied(HttpServer.Request request, Authentication authentication) { - if (authentication != null && totp.isEnabled(authentication.getName())) { + if (requires(authentication)) { forget(asciiLower(authentication.getName()), from(request)); } } @@ -203,6 +229,14 @@ public HttpServer.Response doFilter(HttpServer.Request request, FilterChain chai String code = Responses.param(request, codeParameter); String who = asciiLower(user); String from = from(request); + if (Responses.param(request, "send") != null) { + // Not counted here: the service mails no more than one code in its + // own interval, whoever asks and however often. + if (mailed(user)) { + emailCodes.send(user); + } + return Responses.redirect(page + "?sent"); + } // What is written as a recovery code is counted apart from the one-time // codes, and only at this network: it is the way in for a user whose // one-time codes somebody else has used up the attempts at, so those @@ -225,7 +259,10 @@ public HttpServer.Response doFilter(HttpServer.Request request, FilterChain chai if (!recovery && attempts != null && !attempts.tryAcquire(counted)) { return tooMany(attempts.retryAfterSeconds(counted)); } - boolean accepted = code != null && (totp.verify(user, code) + // The app first: six digits it takes never reach the mailed code, + // which counts every try made against it. + boolean accepted = code != null && ((app(user) && totp.verify(user, code)) + || (mailed(user) && emailCodes.verify(user, code)) || (recoveryCodes != null && recoveryCodes.consume(user, code))); if (!accepted) { return Responses.redirect(page + "?error"); @@ -299,6 +336,14 @@ private HttpServer.Response expired(HttpServer.Request request, HttpSession sess private HttpServer.Response page(HttpServer.Request request) { boolean error = request.queryParam("error") != null; CsrfToken token = CsrfFilter.getToken(request); + HttpSession session = request.getSession(false); + Object stored = session == null ? null : session.getAttribute(PENDING); + Object name = stored instanceof Map ? ((Map) stored).get("name") : null; + boolean mailed = name instanceof String && mailed((String) name); + boolean app = name instanceof String && app((String) name); + String asked = !mailed ? "The code from your authenticator app" + : app ? "The code from your authenticator app, or the code we mailed you" + : "The code we mailed you"; StringBuilder html = new StringBuilder(1024); html.append("\n\n\n\n"); html.append("\n"); @@ -308,9 +353,12 @@ private HttpServer.Response page(HttpServer.Request request) { html.append("\">\n

Enter your code

\n"); if (error) { html.append("

That code was not accepted

\n"); + } else if (request.queryParam("sent") != null) { + html.append("

If a code could be sent, it is on its way

\n"); } - html.append("

\n\n

\n"); @@ -319,7 +367,19 @@ private HttpServer.Response page(HttpServer.Request request) { .append(Responses.escape(token.getParameterName())).append("\" value=\"") .append(Responses.escape(token.getToken())).append("\">\n"); } - html.append("\n\n\n\n"); + html.append("\n\n"); + if (mailed) { + html.append("
\n\n"); + if (token != null) { + html.append("\n"); + } + html.append("\n
\n"); + } + html.append("\n\n"); return Responses.html(200, html.toString()); } } diff --git a/vm/backend/src/com/codename1/backend/security/SecuritySchema.java b/vm/backend/src/com/codename1/backend/security/SecuritySchema.java index 2e7a29809f9..2c0304ea72c 100644 --- a/vm/backend/src/com/codename1/backend/security/SecuritySchema.java +++ b/vm/backend/src/com/codename1/backend/security/SecuritySchema.java @@ -81,6 +81,7 @@ /// | 10 | `cn1_rate_limit.window_end` | Expiry-aware cleanup for limiters with different periods | /// | 11 | `cn1_persistent_logins.previous_token_hash` | Concurrent remember-me rotation grace | /// | 12 | `cn1_persistent_logins.user_agent`, `cn1_persistent_logins.ip_address` | The device a remember-me sign-in came from | +/// | 13 | `cn1_one_time_token`, `cn1_mfa_email`, `cn1_verified_email` | [com.codename1.backend.security.mail.JdbcOneTimeTokenRepository], [com.codename1.backend.security.mfa.JdbcEmailCodeRepository], [com.codename1.backend.security.mail.JdbcVerifiedEmailRepository] | /// /// #### Adopting an existing schema /// @@ -120,7 +121,7 @@ public final class SecuritySchema { // The versions of this set. A version is never edited once released: a new // table, or a change to one, is a new version appended in migrations(). - // The next takes 13. Each version is one method below that returns the + // The next takes 14. Each version is one method below that returns the // statements for an engine, a case in Tables.migrate, and a line in // migrations(). @@ -133,7 +134,8 @@ public final class SecuritySchema { "cn1_users", "cn1_authorities", "cn1_api_key", "cn1_persistent_logins", "cn1_mfa_totp", "cn1_mfa_recovery_code", "cn1_rate_limit", "cn1_federated_identity", "cn1_oauth2_registered_client", "cn1_oauth2_authorization", "cn1_oauth2_token", - "cn1_webauthn_user", "cn1_webauthn_credential" + "cn1_webauthn_user", "cn1_webauthn_credential", "cn1_one_time_token", "cn1_mfa_email", + "cn1_verified_email" }; // Null until configured; then the default name -> configured name map. @@ -266,7 +268,7 @@ static String checkIdentifier(String name) { "users and authorities", "api keys", "persistent logins", "second factors", "rate limits", "federated identities", "oauth2 registered clients", "oauth2 authorizations", "passkeys", "rate limit expiry", "remember me rotation grace", - "remember me devices" + "remember me devices", "mail tokens" }; /// The set, for `Migrations.register`. @@ -324,6 +326,7 @@ public void migrate(MigrationContext context) throws IOException { case 10: statements = rateLimitExpiry(d); break; case 11: statements = rememberMeRotationGrace(); break; case 12: statements = rememberMeDevices(d); break; + case 13: statements = mailTokens(d); break; default: throw new IOException("The security schema has no version " + version); } for (String statement : statements) { @@ -493,4 +496,23 @@ private static String[] passkeys(Dialect d) { + moment(d) + ")", "CREATE INDEX " + index("cn1_webauthn_credential", "user") + " ON " + table("cn1_webauthn_credential") + " (user_id)"}; } + + /// What the mailed flows keep: a link's token (only its SHA-256, so a + /// copy of the table opens nothing), the code of a sign-in by mail, and + /// the address a user has proved to be theirs. + private static String[] mailTokens(Dialect d) { + return new String[] { + "CREATE TABLE " + table("cn1_one_time_token") + " (token_hash " + key(d) + + " NOT NULL PRIMARY KEY, purpose " + key(d) + " NOT NULL, username_key " + key(d) + + " NOT NULL, username " + text(d) + ", expires_at " + moment(d) + ")", + "CREATE INDEX " + index("cn1_one_time_token", "user") + " ON " + + table("cn1_one_time_token") + " (username_key, purpose)", + "CREATE TABLE " + table("cn1_mfa_email") + " (username_key " + key(d) + + " NOT NULL PRIMARY KEY, enabled " + flag(d) + ", code_hash " + text(d) + + ", expires_at " + moment(d) + ", attempts " + d.columnType(Dialect.INTEGER) + + " NOT NULL, sent_at " + moment(d) + ")", + "CREATE TABLE " + table("cn1_verified_email") + " (username_key " + key(d) + + " NOT NULL PRIMARY KEY, email " + text(d) + ", proof " + key(d) + + ", verified_at " + moment(d) + ")"}; + } } diff --git a/vm/backend/src/com/codename1/backend/security/mail/EmailAddressResolver.java b/vm/backend/src/com/codename1/backend/security/mail/EmailAddressResolver.java new file mode 100644 index 00000000000..1f56732591c --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/EmailAddressResolver.java @@ -0,0 +1,49 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.mail.Mail; + +/// Says where a user's mail goes. A user record here has a name and no +/// address, so the flows that send mail ask this. +/// +/// ```java +/// @Bean +/// EmailAddressResolver addresses(AccountRepository accounts) { +/// return username -> accounts.findByUsername(username).getEmail(); +/// } +/// ``` +public interface EmailAddressResolver { + /// The user name itself where it is an address, as it is in an + /// application whose users sign in with one; nothing otherwise. + EmailAddressResolver USERNAME = new EmailAddressResolver() { + @Override + public String emailFor(String username) { + return Mail.isAddress(username) ? username : null; + } + }; + + /// The address mail for `username` is sent to, or null when there is none + /// and nothing should be sent. + String emailFor(String username); +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/EmailVerificationService.java b/vm/backend/src/com/codename1/backend/security/mail/EmailVerificationService.java new file mode 100644 index 00000000000..b61cd9142f5 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/EmailVerificationService.java @@ -0,0 +1,220 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.Base64Url; +import com.codename1.backend.Crypto; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.mvc.Model; +import com.codename1.backend.security.Authentication; +import com.codename1.backend.security.AuthorizationDecision; +import com.codename1.backend.security.AuthorizationManager; +import com.codename1.backend.security.Clock; +import com.codename1.backend.security.RequestAuthorizationContext; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; +import java.util.function.Supplier; + +/// Proves that an address is a user's, by mailing it a link. +/// +/// ```java +/// verification.send(username, "ada@example.com", "https://example.com/verify-email"); +/// // ... the link is opened +/// String username = verification.confirm(token); // null unless it is good +/// ``` +/// +/// `http.emailVerification()` adds the address the link opens. An address is +/// proved once and stays so until [#send] is called with another one. +/// +/// The service is an [EmailAddressResolver] too: it answers a user's proved +/// address, so a password reset or a sign-in code goes to an address the user +/// has shown to be theirs. +/// +/// ```java +/// http.authorizeHttpRequests(a -> a +/// .requestMatchers("/billing/**").access(verification.verified()) +/// .anyRequest().authenticated()); +/// ``` +/// +/// The mail is plain by default. Name a template to send the application's +/// own: it is rendered with `link`, `username` and `minutes`. +public final class EmailVerificationService implements EmailAddressResolver { + /// The purpose of the tokens this service gives out. + public static final String PURPOSE = "email-verification"; + + private final OneTimeTokenService tokens; + private final MailSender mail; + private final VerifiedEmailRepository repository; + private Clock clock = Clock.SYSTEM; + private int validityMinutes = 24 * 60; + private String subject = "Confirm your email address"; + private String template; + + public EmailVerificationService(OneTimeTokenService tokens, MailSender mail, + VerifiedEmailRepository repository) { + if (tokens == null || mail == null || repository == null) { + throw new IllegalArgumentException("Verifying an address needs tokens, a mail " + + "sender and a store; none can be null"); + } + this.tokens = tokens; + this.mail = mail; + this.repository = repository; + } + + /// The clock the moment of proof is read from; a test sets its own. + public EmailVerificationService clock(Clock clock) { + this.clock = clock; + return this; + } + + /// How long a link works; a day unless set. + public EmailVerificationService validityMinutes(int validityMinutes) { + if (validityMinutes < 1) { + throw new IllegalArgumentException("validityMinutes must be positive"); + } + this.validityMinutes = validityMinutes; + return this; + } + + /// The subject of the mail. + public EmailVerificationService subject(String subject) { + this.subject = subject; + return this; + } + + /// The template the mail is rendered from, such as `mail/verify-email`, in + /// place of the plain mail. + public EmailVerificationService template(String view) { + this.template = view; + return this; + } + + /// Records `email` as the address `username` claims and mails it a link + /// to `confirmUrl` with a `token` parameter added. Whatever address the + /// user had proved is no longer theirs until this one is. + /// + /// #### Parameters + /// + /// - `confirmUrl`: the whole address the link opens, + /// `https://example.com/verify-email` + /// + /// #### Throws + /// + /// - `IllegalArgumentException`: `email` is not an address + /// - `IOException`: the mail could not be handed over + public void send(String username, String email, String confirmUrl) throws IOException { + if (username == null || !Mail.isAddress(email)) { + throw new IllegalArgumentException("Not an address a mail can be sent to"); + } + // The token first, and the address recorded with a proof made from + // it: the address is proved only by the link that was mailed to it. + // A token belongs to a user, not to an address, so without the proof + // a link mailed to one address would prove whichever the user claimed + // last -- one that is somebody else's. + String token = tokens.create(PURPOSE, username, validityMinutes * 60); + repository.savePending(username, email, proof(token)); + String link = MailText.link(confirmUrl, token); + Mail.Builder message = Mail.to(email).subject(subject); + if (template != null) { + message.template(template, new Model().addAttribute("link", link) + .addAttribute("username", username) + .addAttribute("minutes", Integer.valueOf(validityMinutes))); + } else { + String[] lines = { + // Named, so that somebody who gave another person's address + // for an account of their own is not confirmed by a reader + // who takes the mail for one about theirs. + "This address was given for the account \"" + username + "\".", + "Open this link to confirm that it is yours:", + "If that is not your account, ignore this mail."}; + message.text(MailText.text(lines, link)).html(MailText.html(lines, link, + "Confirm address")); + } + mail.send(message.build()); + } + + /// Takes the token of a link. Answers the user whose address is now + /// proved, or null when the token is not good. + public String confirm(String token) { + String username = tokens.consume(PURPOSE, token); + if (username == null || !repository.markVerified(username, proof(token), + clock.currentTimeMillis())) { + return null; + } + return username; + } + + /// What ties an address to the one link mailed to it: a hash of the + /// token under a label of its own, so it is neither the token nor the + /// hash the token is kept under. + private static String proof(String token) { + return Base64Url.encode(Crypto.sha256(WireBuffer.utf8("email-proof:" + token))); + } + + /// Whether `username` has proved their address. + public boolean isVerified(String username) { + return repository.isVerified(username); + } + + /// The address `username` has proved, or null. + public String getVerifiedEmail(String username) { + return repository.findVerifiedEmail(username); + } + + /// Forgets `username`'s address and any link mailed to it. + public void forget(String username) { + tokens.revoke(PURPOSE, username); + repository.delete(username); + } + + /// The address `username` has proved; failing that, the user name where + /// it is an address. + @Override + public String emailFor(String username) { + String proved = getVerifiedEmail(username); + return proved != null ? proved : EmailAddressResolver.USERNAME.emailFor(username); + } + + /// Grants a request to a signed-in user who has proved their address. + public AuthorizationManager verified() { + return new Verified(this); + } + + private static final class Verified + implements AuthorizationManager { + private final EmailVerificationService service; + + Verified(EmailVerificationService service) { + this.service = service; + } + + @Override + public AuthorizationDecision check(Supplier authentication, + RequestAuthorizationContext object) { + Authentication who = authentication.get(); + return new AuthorizationDecision(who != null && who.isAuthenticated() + && who.getName() != null && service.isVerified(who.getName())); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/InMemoryOneTimeTokenRepository.java b/vm/backend/src/com/codename1/backend/security/mail/InMemoryOneTimeTokenRepository.java new file mode 100644 index 00000000000..2c703fdfa62 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/InMemoryOneTimeTokenRepository.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.security.SecuritySchema; +import java.util.HashMap; +import java.util.Iterator; +import java.util.Map; + +/// Tokens kept in this process: gone when it stops, and unknown to any other +/// server. For development and tests; see [JdbcOneTimeTokenRepository] +/// otherwise. +public final class InMemoryOneTimeTokenRepository implements OneTimeTokenRepository { + private final Map byHash = new HashMap(); + + private static final class Entry { + final String purpose; + final String userKey; + final String username; + final long expiresAt; + + Entry(String purpose, String userKey, String username, long expiresAt) { + this.purpose = purpose; + this.userKey = userKey; + this.username = username; + this.expiresAt = expiresAt; + } + } + + @Override + public synchronized void save(String tokenHash, String purpose, String username, long now, + long expiresAt) { + String user = SecuritySchema.usernameKey(username); + for (Iterator it = byHash.values().iterator(); it.hasNext();) { + Entry e = it.next(); + if (e.expiresAt < now || e.userKey.equals(user) && e.purpose.equals(purpose)) { + it.remove(); + } + } + byHash.put(tokenHash, new Entry(purpose, user, username, expiresAt)); + } + + @Override + public synchronized String consume(String tokenHash, String purpose, long now) { + Entry e = tokenHash == null ? null : byHash.get(tokenHash); + if (e == null || !e.purpose.equals(purpose)) { + return null; + } + byHash.remove(tokenHash); + return e.expiresAt < now ? null : e.username; + } + + @Override + public synchronized void removeAll(String username, String purpose) { + String user = SecuritySchema.usernameKey(username); + for (Iterator it = byHash.values().iterator(); it.hasNext();) { + Entry e = it.next(); + if (e.userKey.equals(user) && e.purpose.equals(purpose)) { + it.remove(); + } + } + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/InMemoryVerifiedEmailRepository.java b/vm/backend/src/com/codename1/backend/security/mail/InMemoryVerifiedEmailRepository.java new file mode 100644 index 00000000000..e8becdf5051 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/InMemoryVerifiedEmailRepository.java @@ -0,0 +1,79 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.security.SecuritySchema; +import java.util.HashMap; +import java.util.Map; + +/// Addresses kept in this process: gone when it stops. For development and +/// tests; see [JdbcVerifiedEmailRepository] otherwise. +public final class InMemoryVerifiedEmailRepository implements VerifiedEmailRepository { + private final Map emails = new HashMap(); + private final Map verified = new HashMap(); + private final Map proofs = new HashMap(); + + @Override + public synchronized void savePending(String username, String email, String proof) { + String user = SecuritySchema.usernameKey(username); + emails.put(user, email); + proofs.put(user, proof); + verified.remove(user); + } + + @Override + public synchronized boolean markVerified(String username, String proof, long now) { + String user = SecuritySchema.usernameKey(username); + String recorded = proofs.get(user); + if (proof == null || recorded == null || !recorded.equals(proof)) { + return false; + } + // Used up with the link it stands for. + proofs.remove(user); + verified.put(user, Long.valueOf(now)); + return true; + } + + @Override + public synchronized String findEmail(String username) { + return username == null ? null : emails.get(SecuritySchema.usernameKey(username)); + } + + @Override + public synchronized String findVerifiedEmail(String username) { + return isVerified(username) ? findEmail(username) : null; + } + + @Override + public synchronized boolean isVerified(String username) { + return username != null && verified.containsKey(SecuritySchema.usernameKey(username)); + } + + @Override + public synchronized void delete(String username) { + String user = SecuritySchema.usernameKey(username); + emails.remove(user); + proofs.remove(user); + verified.remove(user); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/JdbcOneTimeTokenRepository.java b/vm/backend/src/com/codename1/backend/security/mail/JdbcOneTimeTokenRepository.java new file mode 100644 index 00000000000..299594b3736 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/JdbcOneTimeTokenRepository.java @@ -0,0 +1,136 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.DataSource; +import com.codename1.backend.Database; +import com.codename1.backend.security.AuthenticationServiceException; +import com.codename1.backend.security.SecuritySchema; +import java.io.IOException; +import java.util.Map; + +/// Tokens kept in the server's database, in the `cn1_one_time_token` table of +/// [SecuritySchema]. A token is used up by one `DELETE` that names it, and +/// counted as used only when that removed exactly one row, so two servers +/// handed the same link let one of them through. +public final class JdbcOneTimeTokenRepository implements OneTimeTokenRepository { + private static final String TABLE = "cn1_one_time_token"; + private final DataSource dataSource; + + public JdbcOneTimeTokenRepository(DataSource dataSource) { + if (dataSource == null) { + throw new IllegalArgumentException("dataSource cannot be null"); + } + this.dataSource = dataSource; + } + + private static RuntimeException failed(Exception err) { + if (err instanceof RuntimeException) { + return (RuntimeException) err; + } + return new AuthenticationServiceException("The one-time token store could not be " + + "reached: " + err.getMessage(), err); + } + + @Override + public void save(String tokenHash, String purpose, String username, long now, + long expiresAt) { + try { + dataSource.inTransaction(new Store(tokenHash, purpose, username, now, expiresAt)); + } catch (Exception err) { + throw failed(err); + } + } + + /// A user's token for a purpose replaced by a new one, as the body of a + /// transaction. + private static final class Store implements DataSource.Work { + private final String tokenHash; + private final String purpose; + private final String username; + private final long now; + private final long expiresAt; + + Store(String tokenHash, String purpose, String username, long now, long expiresAt) { + this.tokenHash = tokenHash; + this.purpose = purpose; + this.username = username; + this.now = now; + this.expiresAt = expiresAt; + } + + @Override + public Object run(Database db) throws Exception { + String table = SecuritySchema.table(TABLE); + String user = SecuritySchema.usernameKey(username); + db.execute("DELETE FROM " + table + " WHERE expires_at < ?", + new Object[] {Long.valueOf(now)}); + db.execute("DELETE FROM " + table + " WHERE username_key = ? AND purpose = ?", + new Object[] {user, purpose}); + db.execute("INSERT INTO " + table + " (token_hash, purpose, username_key, username, " + + "expires_at) VALUES (?, ?, ?, ?, ?)", new Object[] {tokenHash, purpose, + user, username, Long.valueOf(expiresAt)}); + return null; + } + } + + @Override + public String consume(String tokenHash, String purpose, long now) { + if (tokenHash == null || purpose == null) { + return null; + } + try { + String table = SecuritySchema.table(TABLE); + Map row = dataSource.queryOne("SELECT username, expires_at FROM " + table + + " WHERE token_hash = ? AND purpose = ?", new Object[] {tokenHash, purpose}); + if (row == null) { + return null; + } + // Only the removal decides, across server processes. + if (dataSource.execute("DELETE FROM " + table + " WHERE token_hash = ? AND " + + "purpose = ?", new Object[] {tokenHash, purpose}) != 1) { + return null; + } + Object expires = row.get("expires_at"); + Object username = row.get("username"); + if (!(expires instanceof Number) || ((Number) expires).longValue() < now + || !(username instanceof String)) { + return null; + } + return (String) username; + } catch (IOException err) { + throw failed(err); + } + } + + @Override + public void removeAll(String username, String purpose) { + try { + dataSource.execute("DELETE FROM " + SecuritySchema.table(TABLE) + + " WHERE username_key = ? AND purpose = ?", + new Object[] {SecuritySchema.usernameKey(username), purpose}); + } catch (IOException err) { + throw failed(err); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/JdbcVerifiedEmailRepository.java b/vm/backend/src/com/codename1/backend/security/mail/JdbcVerifiedEmailRepository.java new file mode 100644 index 00000000000..6ccc7128ef0 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/JdbcVerifiedEmailRepository.java @@ -0,0 +1,158 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.DataSource; +import com.codename1.backend.Database; +import com.codename1.backend.security.AuthenticationServiceException; +import com.codename1.backend.security.SecuritySchema; +import java.io.IOException; +import java.util.Map; + +/// Addresses kept in the server's database, in the `cn1_verified_email` table +/// of [SecuritySchema]. `verified_at` is 0 until the address is proved. +public final class JdbcVerifiedEmailRepository implements VerifiedEmailRepository { + private static final String TABLE = "cn1_verified_email"; + private final DataSource dataSource; + + public JdbcVerifiedEmailRepository(DataSource dataSource) { + if (dataSource == null) { + throw new IllegalArgumentException("dataSource cannot be null"); + } + this.dataSource = dataSource; + } + + private static RuntimeException failed(Exception err) { + if (err instanceof RuntimeException) { + return (RuntimeException) err; + } + return new AuthenticationServiceException("The verified address store could not be " + + "reached: " + err.getMessage(), err); + } + + @Override + public void savePending(String username, String email, String proof) { + Store store = new Store(SecuritySchema.usernameKey(username), email, proof); + try { + dataSource.inTransaction(store); + } catch (Exception first) { + // Two sends for one user at once: both removed the row and one + // insert met the other's. Once more, over the row that is there. + try { + dataSource.inTransaction(store); + } catch (Exception err) { + throw failed(err); //NOPMD PreserveStackTrace - the second failure is the one to report + } + } + } + + /// A user's address replaced by an unproved one, as the body of a + /// transaction. + private static final class Store implements DataSource.Work { + private final String user; + private final String email; + private final String proof; + + Store(String user, String email, String proof) { + this.user = user; + this.email = email; + this.proof = proof; + } + + @Override + public Object run(Database db) throws Exception { + String table = SecuritySchema.table(TABLE); + db.execute("DELETE FROM " + table + " WHERE username_key = ?", new Object[] {user}); + db.execute("INSERT INTO " + table + " (username_key, email, proof, verified_at) " + + "VALUES (?, ?, ?, 0)", new Object[] {user, email, proof}); + return null; + } + } + + @Override + public boolean markVerified(String username, String proof, long now) { + if (proof == null) { + return false; + } + try { + // One statement, so of two callers one is answered. The count is + // of rows changed on MySQL and of rows matched elsewhere; a + // moment is never 0, so a matched row always changes. The proof + // goes with the link it stands for. + return dataSource.execute("UPDATE " + SecuritySchema.table(TABLE) + + " SET verified_at = ?, proof = NULL WHERE username_key = ? AND proof = ?" + + " AND verified_at = 0", + new Object[] {Long.valueOf(now), SecuritySchema.usernameKey(username), + proof}) == 1; + } catch (IOException err) { + throw failed(err); + } + } + + private Map row(String username) { + if (username == null) { + return null; + } + try { + return dataSource.queryOne("SELECT email, verified_at FROM " + + SecuritySchema.table(TABLE) + " WHERE username_key = ?", + new Object[] {SecuritySchema.usernameKey(username)}); + } catch (IOException err) { + throw failed(err); + } + } + + @Override + public String findEmail(String username) { + Map row = row(username); + Object email = row == null ? null : row.get("email"); + return email instanceof String ? (String) email : null; + } + + @Override + public String findVerifiedEmail(String username) { + Map row = row(username); + Object at = row == null ? null : row.get("verified_at"); + Object email = row == null ? null : row.get("email"); + return at instanceof Number && ((Number) at).longValue() > 0 + && email instanceof String ? (String) email : null; + } + + @Override + public boolean isVerified(String username) { + Map row = row(username); + Object at = row == null ? null : row.get("verified_at"); + return at instanceof Number && ((Number) at).longValue() > 0; + } + + @Override + public void delete(String username) { + try { + dataSource.execute("DELETE FROM " + SecuritySchema.table(TABLE) + + " WHERE username_key = ?", + new Object[] {SecuritySchema.usernameKey(username)}); + } catch (IOException err) { + throw failed(err); + } + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/MailText.java b/vm/backend/src/com/codename1/backend/security/mail/MailText.java new file mode 100644 index 00000000000..910201918b3 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/MailText.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +/// The plain mails the flows send when the application names no template. +final class MailText { + private MailText() { + } + + /// `url` with `token` as a parameter. + static String link(String url, String token) { + if (url == null || !(url.regionMatches(true, 0, "https://", 0, 8) + || url.regionMatches(true, 0, "http://", 0, 7))) { + throw new IllegalArgumentException("A mailed link needs a whole address, starting " + + "with https://: " + url); + } + return url + (url.indexOf('?') < 0 ? '?' : '&') + "token=" + token; + } + + /// Three lines with the link after the second. + static String text(String[] lines, String link) { + return lines[0] + "\r\n\r\n" + lines[1] + "\r\n\r\n" + link + "\r\n\r\n" + lines[2] + + "\r\n"; + } + + static String html(String[] lines, String link, String label) { + String href = escape(link); + return "\n\n\n

" + escape(lines[0]) + "

\n

" + + escape(lines[1]) + "

\n

" + escape(label) + + "

\n

" + href + "

\n

" + escape(lines[2]) + + "

\n\n\n"; + } + + static String escape(String value) { + StringBuilder sb = new StringBuilder(value.length() + 8); + for (int iter = 0 ; iter < value.length() ; iter++) { + char c = value.charAt(iter); + switch (c) { + case '&': sb.append("&"); break; + case '<': sb.append("<"); break; + case '>': sb.append(">"); break; + case '"': sb.append("""); break; + case '\'': sb.append("'"); break; + default: sb.append(c); + } + } + return sb.toString(); + } + + /// Notes a mail that could not be handed over; never whom it was for. + static void notSent(String what, Exception err) { + System.err.println("cn1: a " + what + " mail was not sent: " + err.getMessage()); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenRepository.java b/vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenRepository.java new file mode 100644 index 00000000000..804ebb4ecac --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenRepository.java @@ -0,0 +1,48 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +/// Where [OneTimeTokenService] keeps the tokens it has given out: never a +/// token, only its hash. +public interface OneTimeTokenRepository { + /// Keeps a token for `username`, in place of any that user had for + /// `purpose`: the newest link is the only one that works. + /// + /// #### Parameters + /// + /// - `tokenHash`: the token's hash + /// - `purpose`: what the token may be used for + /// - `username`: whose it is + /// - `now`: the moment, epoch milliseconds; tokens that expired before it + /// may be dropped + /// - `expiresAt`: when the token stops working, epoch milliseconds + void save(String tokenHash, String purpose, String username, long now, long expiresAt); + + /// Uses a token up. Answers the user it was given to, or null when there + /// is none for `purpose`, it has expired, or another caller used it first: + /// of two callers with one token, one is answered. + String consume(String tokenHash, String purpose, long now); + + /// Drops every token `username` has for `purpose`. + void removeAll(String username, String purpose); +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenService.java b/vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenService.java new file mode 100644 index 00000000000..0b05218f973 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/OneTimeTokenService.java @@ -0,0 +1,101 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.Base64Url; +import com.codename1.backend.Crypto; +import com.codename1.backend.security.Clock; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; + +/// Gives out tokens that work once: what a mailed link carries. +/// +/// ```java +/// String token = tokens.create("invite", "ada@example.com", 3600); +/// // ... later, from the link +/// String username = tokens.consume("invite", token); // null unless it is good +/// ``` +/// +/// A token is 256 random bits, written in 43 characters that need no escaping +/// in a URL. It is for one purpose and one user, stops working at its expiry +/// and when it is used, and is replaced when the same user is given another +/// for the same purpose. Only its SHA-256 is kept: whoever reads the table +/// learns no token, and a token is looked up by that hash, so how long a +/// lookup takes says nothing about a token that was guessed nearly right. +public final class OneTimeTokenService { + private final OneTimeTokenRepository repository; + private Clock clock = Clock.SYSTEM; + + public OneTimeTokenService(OneTimeTokenRepository repository) { + if (repository == null) { + throw new IllegalArgumentException("repository cannot be null"); + } + this.repository = repository; + } + + /// The clock expiry is measured by; a test sets its own. + public OneTimeTokenService clock(Clock clock) { + this.clock = clock; + return this; + } + + /// A new token for `username`, good for `validitySeconds` and for + /// `purpose` alone. Any earlier token of that user and purpose stops + /// working. + public String create(String purpose, String username, int validitySeconds) { + if (purpose == null || purpose.length() == 0 || username == null) { + throw new IllegalArgumentException("A token needs a purpose and a user"); + } + if (validitySeconds < 1) { + throw new IllegalArgumentException("validitySeconds must be positive"); + } + String token; + try { + token = Base64Url.encode(Crypto.randomBytes(32)); + } catch (IOException err) { + throw new IllegalStateException("No random source: " + err.getMessage(), err); + } + long now = clock.currentTimeMillis(); + repository.save(hash(token), purpose, username, now, now + validitySeconds * 1000L); + return token; + } + + /// Uses `token` up and answers the user it was given to, or null when it + /// is not a token for `purpose`, has expired or was already used. + public String consume(String purpose, String token) { + // 43 characters is what create() writes; anything else was never ours. + if (purpose == null || token == null || token.length() != 43) { + return null; + } + return repository.consume(hash(token), purpose, clock.currentTimeMillis()); + } + + /// Stops whatever token `username` holds for `purpose` from working. + public void revoke(String purpose, String username) { + repository.removeAll(username, purpose); + } + + private static String hash(String token) { + return Base64Url.encode(Crypto.sha256(WireBuffer.utf8(token))); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/PasswordResetService.java b/vm/backend/src/com/codename1/backend/security/mail/PasswordResetService.java new file mode 100644 index 00000000000..596d94e6752 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/PasswordResetService.java @@ -0,0 +1,231 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.mvc.Model; +import com.codename1.backend.security.core.userdetails.UserDetails; +import com.codename1.backend.security.core.userdetails.UserDetailsPasswordService; +import com.codename1.backend.security.core.userdetails.UserDetailsService; +import com.codename1.backend.security.core.userdetails.UsernameNotFoundException; +import com.codename1.backend.security.crypto.PasswordEncoder; +import com.codename1.backend.security.rememberme.PersistentTokenRepository; +import java.io.IOException; + +/// Resets a forgotten password by mail. +/// +/// ```java +/// @Bean +/// PasswordResetService passwordReset(OneTimeTokenService tokens, MailSender mail, +/// JdbcUserDetailsManager users, PasswordEncoder encoder) { +/// return new PasswordResetService(tokens, mail, users, users, encoder); +/// } +/// ``` +/// +/// [#request] mails a link that works once and for half an hour; [#reset] +/// takes the token from that link and the new password. `http.passwordReset()` +/// puts both behind pages; an application with pages of its own calls the two +/// methods from its controllers. +/// +/// [#request] returns the same way whether or not there is such a user, and +/// whether or not the mail could be sent, so that asking tells nobody which +/// accounts exist. The mail is handed to the server's mail queue, so the call +/// never waits for a mail server. For an account that exists it still makes a +/// token and stores it, which somebody timing many requests could measure: +/// limit how often it can be called. The pages do, a controller of the +/// application's own has to. +/// +/// A reset removes the user's remember-me sign-ins when the service is given +/// their store. A session that is already signed in stays so until it ends. +/// +/// The mail is plain by default. Name a template to send the application's +/// own: it is rendered with `link`, `username` and `minutes`. +public final class PasswordResetService { + /// The purpose of the tokens this service gives out. + public static final String PURPOSE = "password-reset"; + + private final OneTimeTokenService tokens; + private final MailSender mail; + private final UserDetailsService users; + private final UserDetailsPasswordService passwords; + private final PasswordEncoder encoder; + private EmailAddressResolver addresses = EmailAddressResolver.USERNAME; + private PersistentTokenRepository rememberMe; + private int validityMinutes = 30; + private int minimumLength = 8; + private String subject = "Reset your password"; + private String template; + + public PasswordResetService(OneTimeTokenService tokens, MailSender mail, + UserDetailsService users, UserDetailsPasswordService passwords, + PasswordEncoder encoder) { + if (tokens == null || mail == null || users == null || passwords == null + || encoder == null) { + throw new IllegalArgumentException("A password reset needs tokens, a mail sender, " + + "the users, their password store and an encoder; none can be null"); + } + this.tokens = tokens; + this.mail = mail; + this.users = users; + this.passwords = passwords; + this.encoder = encoder; + } + + /// Where each user's mail goes. Without one, a user name that is an + /// address is used and other users are sent nothing. + public PasswordResetService addressResolver(EmailAddressResolver addressResolver) { + this.addresses = addressResolver == null ? EmailAddressResolver.USERNAME + : addressResolver; + return this; + } + + /// The remember-me sign-ins to remove when a password is reset. + public PasswordResetService rememberMeTokens(PersistentTokenRepository repository) { + this.rememberMe = repository; + return this; + } + + /// How long a link works; 30 minutes unless set. + public PasswordResetService validityMinutes(int validityMinutes) { + if (validityMinutes < 1) { + throw new IllegalArgumentException("validityMinutes must be positive"); + } + this.validityMinutes = validityMinutes; + return this; + } + + /// The shortest password [#reset] takes; 8 unless set. + public PasswordResetService minimumLength(int minimumLength) { + this.minimumLength = minimumLength < 1 ? 1 : minimumLength; + return this; + } + + /// The subject of the mail. + public PasswordResetService subject(String subject) { + this.subject = subject; + return this; + } + + /// The template the mail is rendered from, such as `mail/password-reset`, + /// in place of the plain mail. + public PasswordResetService template(String view) { + this.template = view; + return this; + } + + /// The shortest password [#reset] takes. + public int getMinimumLength() { + return minimumLength; + } + + /// Mails `username` a link to `resetUrl` with a `token` parameter added. + /// Does nothing, and says nothing, when there is no such user, the + /// account is disabled or it has no address. + /// + /// #### Parameters + /// + /// - `username`: the account, as the person asking typed it + /// - `resetUrl`: the whole address of the page that takes the new + /// password, `https://example.com/reset-password`. Build it from the + /// server's configured address, never from the request's `Host` header: + /// that header is the caller's to choose, and the link is mailed to + /// somebody else. + public void request(String username, String resetUrl) { + if (username == null || username.length() == 0 || username.length() > 320) { + return; + } + UserDetails user; + try { + user = users.loadUserByUsername(username); + } catch (UsernameNotFoundException none) { + return; + } + if (user == null || !user.isEnabled()) { + return; + } + String address = addresses.emailFor(user.getUsername()); + if (address == null || !Mail.isAddress(address)) { + return; + } + String token = tokens.create(PURPOSE, user.getUsername(), validityMinutes * 60); + String link = MailText.link(resetUrl, token); + try { + Mail.Builder message = Mail.to(address).subject(subject); + if (template != null) { + message.template(template, new Model().addAttribute("link", link) + .addAttribute("username", user.getUsername()) + .addAttribute("minutes", Integer.valueOf(validityMinutes))); + } else { + String[] lines = { + "Somebody asked to reset the password of your account.", + "Open this link within " + validityMinutes + " minutes to choose a new one:", + "If this was not you, ignore this mail: your password has not changed."}; + message.text(MailText.text(lines, link)).html(MailText.html(lines, link, + "Reset password")); + } + mail.send(message.build()); + } catch (IOException err) { + // Not thrown: an error for the accounts that exist tells them apart. + MailText.notSent("password reset", err); + } catch (RuntimeException err) { + MailText.notSent("password reset", err); + } + } + + /// Whether `password` is one [#reset] would take. + public boolean isAcceptable(String password) { + return password != null && password.length() >= minimumLength + && password.length() <= 1024; + } + + /// Sets the password of the user `token` was mailed to. False when the + /// token is not good -- wrong, expired or used -- or the password is too + /// short. A password that is too short leaves the token good for another + /// try. A token is used up before the password is stored, so that it + /// cannot set two: if the store then fails, this throws what the store + /// threw and the user asks for another link. + public boolean reset(String token, String newPassword) { + if (!isAcceptable(newPassword)) { + return false; + } + String username = tokens.consume(PURPOSE, token); + if (username == null) { + return false; + } + UserDetails user; + try { + user = users.loadUserByUsername(username); + } catch (UsernameNotFoundException gone) { + return false; + } + if (user == null || !user.isEnabled()) { + return false; + } + passwords.updatePassword(user, encoder.encode(newPassword)); + if (rememberMe != null) { + rememberMe.removeUserTokens(user.getUsername()); + } + return true; + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/VerifiedEmailRepository.java b/vm/backend/src/com/codename1/backend/security/mail/VerifiedEmailRepository.java new file mode 100644 index 00000000000..930d1e360bc --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/VerifiedEmailRepository.java @@ -0,0 +1,56 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mail; + +/// Where [EmailVerificationService] keeps each user's address and whether it +/// has been proved. +public interface VerifiedEmailRepository { + /// Records `email` as the address `username` claims, not yet proved, in + /// place of whatever was recorded -- a proved address included: a user + /// has one address, and it is the one they gave last. An application + /// that wants the old one to stand until the new one is proved keeps it + /// in its own table and calls this when it lets go of it. `proof` is what [#markVerified] has to + /// be given for this address: it stands for the one link that was mailed + /// there, so a link mailed to an earlier address proves nothing about + /// this one. + void savePending(String username, String email, String proof); + + /// Marks the address `username` claims as proved at `now`, when `proof` + /// is the one it was recorded with. False when the user claims none, or + /// claims one recorded with another proof. + boolean markVerified(String username, String proof, long now); + + /// The address `username` claims, proved or not, or null. + String findEmail(String username); + + /// The address `username` has proved, or null when they claim none or + /// have not proved the one they claim. One read: an address replaced + /// between two reads would be answered as proved when it is not. + String findVerifiedEmail(String username); + + /// Whether the address `username` claims has been proved. + boolean isVerified(String username); + + /// Forgets `username`'s address. + void delete(String username); +} diff --git a/vm/backend/src/com/codename1/backend/security/mail/package-info.java b/vm/backend/src/com/codename1/backend/security/mail/package-info.java new file mode 100644 index 00000000000..76c8d95a0c5 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mail/package-info.java @@ -0,0 +1,41 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +/// What a server does by mail on a user's behalf: resetting a forgotten +/// password and proving that an address is theirs. +/// +/// Both mail a link carrying a token from [OneTimeTokenService]: 256 random +/// bits that work once, for one purpose, until they expire. Only the token's +/// SHA-256 is kept, so a copy of the table opens nothing. +/// +/// - [PasswordResetService] mails the link and sets the new password; +/// `http.passwordReset(...)` adds the two pages in front of it. +/// - [EmailVerificationService] mails the link and records the address as +/// proved; `http.emailVerification(...)` adds the address the link opens. +/// +/// A code mailed as a second factor at sign-in is +/// [com.codename1.backend.security.mfa.EmailCodeService]. +/// +/// Everything here sends through [com.codename1.backend.mail.MailSender], so +/// it goes wherever `cn1.mail.url` says, and a test reads what was sent from +/// `MailSender.sent()`. +package com.codename1.backend.security.mail; diff --git a/vm/backend/src/com/codename1/backend/security/mfa/EmailCode.java b/vm/backend/src/com/codename1/backend/security/mfa/EmailCode.java new file mode 100644 index 00000000000..08dc0999572 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mfa/EmailCode.java @@ -0,0 +1,66 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mfa; + +/// What an [EmailCodeRepository] holds for one user. +public final class EmailCode { + private final boolean enabled; + private final String codeHash; + private final long expiresAt; + private final int attempts; + private final long sentAt; + + public EmailCode(boolean enabled, String codeHash, long expiresAt, int attempts, + long sentAt) { + this.enabled = enabled; + this.codeHash = codeHash == null ? "" : codeHash; + this.expiresAt = expiresAt; + this.attempts = attempts; + this.sentAt = sentAt; + } + + /// Whether the user signs in with a mailed code. + public boolean isEnabled() { + return enabled; + } + + /// The hash of the code that is waiting to be entered; empty when none is. + public String getCodeHash() { + return codeHash; + } + + /// When the waiting code stops working, epoch milliseconds. + public long getExpiresAt() { + return expiresAt; + } + + /// How many times a code has been tried against the waiting one. + public int getAttempts() { + return attempts; + } + + /// When the waiting code was mailed, epoch milliseconds. + public long getSentAt() { + return sentAt; + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mfa/EmailCodeRepository.java b/vm/backend/src/com/codename1/backend/security/mfa/EmailCodeRepository.java new file mode 100644 index 00000000000..6c40fd877ac --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mfa/EmailCodeRepository.java @@ -0,0 +1,60 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mfa; + +/// Where [EmailCodeService] keeps who signs in with a mailed code, and the +/// hash of the code each is waiting to enter. +public interface EmailCodeRepository { + /// What is held for `username`, or null. + EmailCode find(String username); + + /// Turns the mailed code on or off for `username`. Turning it off drops + /// any code that is waiting. + void setEnabled(String username, boolean enabled); + + /// Keeps a new code for `username` in place of the one waiting, when the + /// user signs in with a mailed code and the last was sent at or before + /// `sentBefore` -- or after `sentAt`, which is a clock that was set back. + /// Of several callers at once one is answered true, so one mail leaves. + /// + /// The attempts counted against the code it replaces are counted against + /// this one too, unless that code had run out or been used: asking for + /// another is not a way to more tries. And a code with `max` attempts + /// counted is not replaced before it runs out, so the wait that follows + /// them ends at a moment that asking again does not move. + boolean saveCode(String username, String codeHash, long expiresAt, long sentAt, + long sentBefore, int max); + + /// Counts one attempt against the waiting code. False when `max` have + /// been counted already; of several callers at once, no more than `max` + /// are answered true. + boolean countAttempt(String username, int max); + + /// Uses the waiting code up, if it is still the one with `codeHash`. Of + /// two callers with the same code, one is answered true. `signedIn` says + /// the code was typed in and accepted, which also forgets when it was + /// sent: the next sign-in is then mailed a code at once. A code cleared + /// because its mail failed keeps the time, so a failing send is still + /// spaced out. + boolean clear(String username, String codeHash, boolean signedIn); +} diff --git a/vm/backend/src/com/codename1/backend/security/mfa/EmailCodeService.java b/vm/backend/src/com/codename1/backend/security/mfa/EmailCodeService.java new file mode 100644 index 00000000000..fc21a9f2c37 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mfa/EmailCodeService.java @@ -0,0 +1,279 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mfa; + +import com.codename1.backend.Base64Url; +import com.codename1.backend.Crypto; +import com.codename1.backend.mail.Mail; +import com.codename1.backend.mail.MailSender; +import com.codename1.backend.mvc.Model; +import com.codename1.backend.security.Clock; +import com.codename1.backend.security.SecuritySchema; +import com.codename1.backend.security.mail.EmailAddressResolver; +import com.codename1.impl.backend.WireBuffer; +import java.io.IOException; + +/// A code mailed to the user as the second step of signing in. +/// +/// ```java +/// @Bean +/// EmailCodeService emailCodes(DataSource dataSource, MailSender mail) { +/// return new EmailCodeService(new JdbcEmailCodeRepository(dataSource), mail); +/// } +/// ``` +/// +/// With the bean declared, `http.mfa(...)` asks every user for whom +/// [#enable] was called for a code after their password, and mails it. A user +/// who has an authenticator app too is asked for either, and the mail is sent +/// when they ask for it. +/// +/// A code is six digits, works once and for ten minutes, and is given up on +/// after five wrong tries: those five in a million are what protect it, so +/// the count is kept with the code and is the same count on every server. +/// Asking for another code does not start the count again: it is carried to +/// the new code until a code is entered right or runs out, and once the five +/// are used no code is mailed until then: the user waits out what is left of +/// the ten minutes, and asking again does not make the wait longer. Only a hash of the code is +/// kept. Another is not mailed within half a minute of the last. +/// +/// A mailed code is as strong as the mailbox it goes to, and weaker than an +/// authenticator app or a passkey: offer it as the easy option, not the only +/// one. +/// +/// The mail is plain by default. Name a template to send the application's +/// own: it is rendered with `code`, `username` and `minutes`. +public final class EmailCodeService { + private final EmailCodeRepository repository; + private final MailSender mail; + private EmailAddressResolver addresses = EmailAddressResolver.USERNAME; + private Clock clock = Clock.SYSTEM; + private int validitySeconds = 600; + private int resendSeconds = 30; + private int maxAttempts = 5; + private String subject = "Your sign-in code"; + private String template; + + public EmailCodeService(EmailCodeRepository repository, MailSender mail) { + if (repository == null || mail == null) { + throw new IllegalArgumentException("A mailed code needs a store and a mail sender; " + + "neither can be null"); + } + this.repository = repository; + this.mail = mail; + } + + /// Where each user's mail goes. Without one, a user name that is an + /// address is used and other users are sent nothing. + public EmailCodeService addressResolver(EmailAddressResolver addressResolver) { + this.addresses = addressResolver == null ? EmailAddressResolver.USERNAME + : addressResolver; + return this; + } + + /// The clock expiry is measured by; a test sets its own. + public EmailCodeService clock(Clock clock) { + this.clock = clock; + return this; + } + + /// How long a code works; 600 seconds unless set. + public EmailCodeService validitySeconds(int validitySeconds) { + if (validitySeconds < 1) { + throw new IllegalArgumentException("validitySeconds must be positive"); + } + this.validitySeconds = validitySeconds; + return this; + } + + /// How long after one mail the next may be sent; 30 seconds unless set. + public EmailCodeService resendSeconds(int resendSeconds) { + this.resendSeconds = resendSeconds < 0 ? 0 : resendSeconds; + return this; + } + + /// How many tries a user is given, over however many codes, before one + /// is entered right or the last runs out; 5 unless set. + public EmailCodeService maxAttempts(int maxAttempts) { + if (maxAttempts < 1) { + throw new IllegalArgumentException("maxAttempts must be positive"); + } + this.maxAttempts = maxAttempts; + return this; + } + + /// The subject of the mail. + public EmailCodeService subject(String subject) { + this.subject = subject; + return this; + } + + /// The template the mail is rendered from, such as `mail/sign-in-code`, in + /// place of the plain mail. + public EmailCodeService template(String view) { + this.template = view; + return this; + } + + /// Whether `username` signs in with a mailed code. + public boolean isEnabled(String username) { + EmailCode held = repository.find(username); + return held != null && held.isEnabled(); + } + + /// Makes `username` sign in with a mailed code from now on. Call it once + /// the user's address is known to be theirs. + public void enable(String username) { + repository.setEnabled(username, true); + } + + /// Stops asking `username` for a mailed code. + public void disable(String username) { + repository.setEnabled(username, false); + } + + /// Mails `username` a new code, which replaces any that was waiting. + /// False, and nothing sent, when the user does not sign in with a mailed + /// code, the last was mailed too recently, the tries were used up and + /// the code they were used on has not run out, the user has no address, + /// or the mail could not be handed over. + public boolean send(String username) { + if (username == null) { + return false; + } + String address = addresses.emailFor(username); + if (address == null || !Mail.isAddress(address)) { + return false; + } + long now = clock.currentTimeMillis(); + String code = newCode(); + String hash = hash(username, code); + // The store decides whether one is due, in the step that keeps the + // code: two requests at once would each find none sent and each mail + // one. + if (!repository.saveCode(username, hash, now + validitySeconds * 1000L, now, + now - resendSeconds * 1000L, maxAttempts)) { + return false; + } + int minutes = (validitySeconds + 59) / 60; + try { + Mail.Builder message = Mail.to(address).subject(subject); + if (template != null) { + message.template(template, new Model().addAttribute("code", code) + .addAttribute("username", username) + .addAttribute("minutes", Integer.valueOf(minutes))); + } else { + message.text("Your sign-in code is " + code + "\r\n\r\nIt works for " + minutes + + " minutes. If you are not signing in, somebody has your password: " + + "change it.\r\n"); + } + mail.send(message.build()); + return true; + } catch (IOException err) { + return notSent(username, hash, err); + } catch (RuntimeException err) { + // A template that throws, an address the mail refuses: the + // sign-in goes on without this mail, it does not end in an error. + return notSent(username, hash, err); + } + } + + private boolean notSent(String username, String hash, Exception err) { + System.err.println("cn1: a sign-in code mail was not sent: " + err.getMessage()); + // Nobody has the code, so it is not left to be guessed at. + repository.clear(username, hash, false); + return false; + } + + /// Whether `code` is the one mailed to `username`. A right code is used + /// up; a wrong one is counted against the code that is waiting. + public boolean verify(String username, String code) { + String digits = normalize(code); + if (username == null || digits == null) { + return false; + } + EmailCode held = repository.find(username); + if (held == null || !held.isEnabled() || held.getCodeHash().length() == 0 + || held.getExpiresAt() < clock.currentTimeMillis()) { + return false; + } + // Counted before it is compared, so that guesses sent together cannot + // each be the last one allowed. + if (!repository.countAttempt(username, maxAttempts)) { + return false; + } + String hash = hash(username, digits); + if (!Crypto.equalsConstantTime(WireBuffer.utf8(hash), + WireBuffer.utf8(held.getCodeHash()))) { + return false; + } + return repository.clear(username, hash, true); + } + + /// Whether `code` is written as a mailed code is: six digits. + public static boolean isCodeShaped(String code) { + return normalize(code) != null; + } + + private static String normalize(String code) { + if (code == null) { + return null; + } + StringBuilder digits = new StringBuilder(6); + for (int iter = 0 ; iter < code.length() ; iter++) { + char c = code.charAt(iter); + if (c == ' ' || c == '-') { + continue; + } + if (c < '0' || c > '9' || digits.length() == 6) { + return null; + } + digits.append(c); + } + return digits.length() == 6 ? digits.toString() : null; + } + + /// Six digits, each as likely as the next. + private static String newCode() { + try { + // A value that would favour the low codes is passed over. + long limit = 0x100000000L - 0x100000000L % 1000000L; + while (true) { + byte[] r = Crypto.randomBytes(4); + long value = ((r[0] & 0xffL) << 24) | ((r[1] & 0xffL) << 16) + | ((r[2] & 0xffL) << 8) | (r[3] & 0xffL); + if (value < limit) { + String text = String.valueOf(1000000L + value % 1000000L); + return text.substring(1); + } + } + } catch (IOException err) { + throw new IllegalStateException("No random source: " + err.getMessage(), err); + } + } + + /// Bound to the user, so that one user's row is no use against another. + private static String hash(String username, String code) { + return Base64Url.encode(Crypto.sha256(WireBuffer.utf8( + SecuritySchema.usernameKey(username) + ":" + code))); + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mfa/InMemoryEmailCodeRepository.java b/vm/backend/src/com/codename1/backend/security/mfa/InMemoryEmailCodeRepository.java new file mode 100644 index 00000000000..b47dd2c3274 --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mfa/InMemoryEmailCodeRepository.java @@ -0,0 +1,83 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mfa; + +import com.codename1.backend.security.SecuritySchema; +import java.util.HashMap; +import java.util.Map; + +/// Mailed codes kept in this process: gone when it stops. For development and +/// tests; see [JdbcEmailCodeRepository] otherwise. +public final class InMemoryEmailCodeRepository implements EmailCodeRepository { + private final Map byUser = new HashMap(); + + @Override + public synchronized EmailCode find(String username) { + return username == null ? null : byUser.get(SecuritySchema.usernameKey(username)); + } + + @Override + public synchronized void setEnabled(String username, boolean enabled) { + byUser.put(SecuritySchema.usernameKey(username), new EmailCode(enabled, "", 0, 0, 0)); + } + + @Override + public synchronized boolean saveCode(String username, String codeHash, long expiresAt, + long sentAt, long sentBefore, int max) { + String user = SecuritySchema.usernameKey(username); + EmailCode was = byUser.get(user); + if (was == null || !was.isEnabled() + || was.getSentAt() > sentBefore && was.getSentAt() <= sentAt + || was.getAttempts() >= max && was.getExpiresAt() >= sentAt) { + return false; + } + byUser.put(user, new EmailCode(true, codeHash, expiresAt, + was.getExpiresAt() < sentAt ? 0 : was.getAttempts(), sentAt)); + return true; + } + + @Override + public synchronized boolean countAttempt(String username, int max) { + String user = SecuritySchema.usernameKey(username); + EmailCode was = byUser.get(user); + if (was == null || was.getAttempts() >= max) { + return false; + } + byUser.put(user, new EmailCode(was.isEnabled(), was.getCodeHash(), was.getExpiresAt(), + was.getAttempts() + 1, was.getSentAt())); + return true; + } + + @Override + public synchronized boolean clear(String username, String codeHash, boolean signedIn) { + String user = SecuritySchema.usernameKey(username); + EmailCode was = byUser.get(user); + if (was == null || codeHash == null || codeHash.length() == 0 + || !codeHash.equals(was.getCodeHash())) { + return false; + } + byUser.put(user, new EmailCode(was.isEnabled(), "", 0, was.getAttempts(), + signedIn ? 0 : was.getSentAt())); + return true; + } +} diff --git a/vm/backend/src/com/codename1/backend/security/mfa/JdbcEmailCodeRepository.java b/vm/backend/src/com/codename1/backend/security/mfa/JdbcEmailCodeRepository.java new file mode 100644 index 00000000000..3203a9ce2fc --- /dev/null +++ b/vm/backend/src/com/codename1/backend/security/mfa/JdbcEmailCodeRepository.java @@ -0,0 +1,170 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.backend.security.mfa; + +import com.codename1.backend.DataSource; +import com.codename1.backend.Database; +import com.codename1.backend.security.AuthenticationServiceException; +import com.codename1.backend.security.SecuritySchema; +import java.io.IOException; +import java.util.Map; + +/// Mailed codes kept in the server's database, in the `cn1_mfa_email` table +/// of [SecuritySchema]. An attempt is counted, and a code used up, by one +/// `UPDATE` whose condition is the limit or the code itself, so servers +/// sharing the table cannot between them allow one attempt too many or take +/// one code twice. +public final class JdbcEmailCodeRepository implements EmailCodeRepository { + private static final String TABLE = "cn1_mfa_email"; + private final DataSource dataSource; + + public JdbcEmailCodeRepository(DataSource dataSource) { + if (dataSource == null) { + throw new IllegalArgumentException("dataSource cannot be null"); + } + this.dataSource = dataSource; + } + + private static RuntimeException failed(Exception err) { + if (err instanceof RuntimeException) { + return (RuntimeException) err; + } + return new AuthenticationServiceException("The mailed code store could not be " + + "reached: " + err.getMessage(), err); + } + + private static EmailCode read(Map row) { + if (row == null) { + return null; + } + Object enabled = row.get("enabled"); + Object hash = row.get("code_hash"); + return new EmailCode(truth(enabled), hash instanceof String ? (String) hash : "", + number(row.get("expires_at")), (int) number(row.get("attempts")), + number(row.get("sent_at"))); + } + + private static boolean truth(Object value) { + return value instanceof Boolean ? ((Boolean) value).booleanValue() + : value instanceof Number && ((Number) value).longValue() != 0; + } + + private static long number(Object value) { + return value instanceof Number ? ((Number) value).longValue() : 0; + } + + @Override + public EmailCode find(String username) { + if (username == null) { + return null; + } + try { + return read(dataSource.queryOne("SELECT enabled, code_hash, expires_at, attempts, " + + "sent_at FROM " + SecuritySchema.table(TABLE) + " WHERE username_key = ?", + new Object[] {SecuritySchema.usernameKey(username)})); + } catch (IOException err) { + throw failed(err); + } + } + + @Override + public void setEnabled(String username, boolean enabled) { + try { + dataSource.inTransaction(new Store(SecuritySchema.usernameKey(username), enabled)); + } catch (Exception err) { + throw failed(err); + } + } + + @Override + public boolean saveCode(String username, String codeHash, long expiresAt, long sentAt, + long sentBefore, int max) { + try { + // One statement, so of several callers one is answered. `attempts` + // is set before `expires_at`: MySQL reads a column a SET has + // already written as its new value, the others as its old one. + return dataSource.execute("UPDATE " + SecuritySchema.table(TABLE) + + " SET attempts = CASE WHEN expires_at < ? THEN 0 ELSE attempts END," + + " code_hash = ?, expires_at = ?, sent_at = ? WHERE username_key = ?" + + " AND enabled = ? AND (sent_at <= ? OR sent_at > ?)" + + " AND (attempts < ? OR expires_at < ?)", + new Object[] {Long.valueOf(sentAt), codeHash, Long.valueOf(expiresAt), + Long.valueOf(sentAt), SecuritySchema.usernameKey(username), + Integer.valueOf(1), Long.valueOf(sentBefore), Long.valueOf(sentAt), + Integer.valueOf(max), Long.valueOf(sentAt)}) == 1; + } catch (IOException err) { + throw failed(err); + } + } + + /// A user's row replaced, as the body of a transaction. Removed and + /// written again rather than updated: an update that changes nothing + /// counts no row on MySQL, which reads as a user with no row. + private static final class Store implements DataSource.Work { + private final String user; + private final boolean enabled; + + Store(String user, boolean enabled) { + this.user = user; + this.enabled = enabled; + } + + @Override + public Object run(Database db) throws Exception { + String table = SecuritySchema.table(TABLE); + db.execute("DELETE FROM " + table + " WHERE username_key = ?", new Object[] {user}); + db.execute("INSERT INTO " + table + " (username_key, enabled, code_hash, " + + "expires_at, attempts, sent_at) VALUES (?, ?, '', 0, 0, 0)", + new Object[] {user, Integer.valueOf(enabled ? 1 : 0)}); + return null; + } + } + + @Override + public boolean countAttempt(String username, int max) { + try { + return dataSource.execute("UPDATE " + SecuritySchema.table(TABLE) + + " SET attempts = attempts + 1 WHERE username_key = ? AND attempts < ?", + new Object[] {SecuritySchema.usernameKey(username), Integer.valueOf(max)}) + == 1; + } catch (IOException err) { + throw failed(err); + } + } + + @Override + public boolean clear(String username, String codeHash, boolean signedIn) { + if (username == null || codeHash == null || codeHash.length() == 0) { + return false; + } + try { + return dataSource.execute("UPDATE " + SecuritySchema.table(TABLE) + + " SET code_hash = '', expires_at = 0" + + (signedIn ? ", sent_at = 0" : "") + " WHERE username_key = ? AND " + + "code_hash = ?", new Object[] {SecuritySchema.usernameKey(username), + codeHash}) == 1; + } catch (IOException err) { + throw failed(err); + } + } +} diff --git a/vm/backend/src/com/codename1/impl/backend/MailRuntime.java b/vm/backend/src/com/codename1/impl/backend/MailRuntime.java new file mode 100644 index 00000000000..fed1d80c071 --- /dev/null +++ b/vm/backend/src/com/codename1/impl/backend/MailRuntime.java @@ -0,0 +1,91 @@ +/* + * Copyright (c) 2012, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.impl.backend; + +import com.codename1.backend.mvc.Model; +import java.io.IOException; +import java.util.concurrent.atomic.AtomicReference; + +/// What the generated wiring gives the mail support, the compiled templates, +/// and what the mail support gives the management endpoint, its state. +/// +/// Both go through here so that neither side names the other: a server that +/// sends no mail links no mail code for the sake of its health report. +/// +/// The templates of an application compile into a class the build generates, +/// which the framework cannot name. The wiring installs a renderer over it +/// for an application that has mail templates; one without them installs +/// nothing and links no template code for mail. +public final class MailRuntime { + private static final AtomicReference RENDERER = + new AtomicReference(); + + private static final AtomicReference PROBE = new AtomicReference(); + + private MailRuntime() { + } + + /// The state of an open mail support, read by the health report and the + /// outbox gauge. + public interface Probe { + /// The transport's name, such as `smtp`. + String transport(); + + /// Whether the transport reached its server the last time it tried. + boolean isHealthy(); + + /// Mails staged and not yet handed to the transport. + int pending(); + } + + /// Called by the mail support when it opens, never by application code. + public static void opened(Probe probe) { + PROBE.set(probe); + } + + /// Called by the mail support when it closes. + public static void closed(Probe probe) { + PROBE.compareAndSet(probe, null); + } + + /// The open mail support, or null when this server has sent no mail. + public static Probe probe() { + return PROBE.get(); + } + + /// Renders a mail's HTML from the application's compiled templates. + public interface TemplateRenderer { + /// The HTML of `templates/.html` as UTF-8. + byte[] render(String view, Model model) throws IOException; + } + + /// Called by the generated wiring, never by application code. + public static void install(TemplateRenderer renderer) { + RENDERER.set(renderer); + } + + /// The installed renderer, or null. + public static TemplateRenderer renderer() { + return RENDERER.get(); + } +} diff --git a/vm/backend/src/com/codename1/impl/backend/Management.java b/vm/backend/src/com/codename1/impl/backend/Management.java index 0a07dbcf771..7f5555c983b 100644 --- a/vm/backend/src/com/codename1/impl/backend/Management.java +++ b/vm/backend/src/com/codename1/impl/backend/Management.java @@ -320,6 +320,7 @@ private HttpServer.Response health(HttpServer.Request request) { out.put("database", db); } keyValue(out); + mail(out); } else { // Not attached yet: the server is still being put together. up = false; @@ -347,6 +348,22 @@ private void keyValue(Map out) { out.put("keyValue", kv); } + /// The mail transport's state, once this server has sent a mail. Reported, + /// not judged, for the reason the key/value store is: mail that cannot leave + /// waits in the outbox and is retried, and a replica is no better at + /// serving requests for being taken out of the load balancer meanwhile. + private void mail(Map out) { + MailRuntime.Probe probe = MailRuntime.probe(); + if (probe == null) { + return; + } + Map mail = new LinkedHashMap(); + mail.put("status", probe.isHealthy() ? "UP" : "DOWN"); + mail.put("transport", probe.transport()); + mail.put("pending", Integer.valueOf(probe.pending())); + out.put("mail", mail); + } + /// Whether the process should be left running, as a liveness probe asks: /// UP while it starts, serves and drains. A draining server is finishing its /// requests, and a probe that restarted it then would cut them off; readiness diff --git a/vm/tests/src/test/java/com/codename1/tools/translator/BackendMailTest.java b/vm/tests/src/test/java/com/codename1/tools/translator/BackendMailTest.java new file mode 100644 index 00000000000..99301b1e5d6 --- /dev/null +++ b/vm/tests/src/test/java/com/codename1/tools/translator/BackendMailTest.java @@ -0,0 +1,132 @@ +/* + * Copyright (c) 2026, Codename One and/or its affiliates. All rights reserved. + * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER. + * This code is free software; you can redistribute it and/or modify it + * under the terms of the GNU General Public License version 2 only, as + * published by the Free Software Foundation. Codename One designates this + * particular file as subject to the "Classpath" exception as provided + * by Oracle in the LICENSE file that accompanied this code. + * + * This code is distributed in the hope that it will be useful, but WITHOUT + * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or + * FITNESS FOR A PARTICULAR PURPOSE. See the GNU General Public License + * version 2 for more details (a copy is included in the LICENSE file that + * accompanied this code). + * + * You should have received a copy of the GNU General Public License version + * 2 along with this work; if not, write to the Free Software Foundation, + * Inc., 51 Franklin St, Fifth Floor, Boston, MA 02110-1301 USA. + * + * Please contact Codename One through http://www.codenameone.com/ if you + * need additional information or have any questions. + */ +package com.codename1.tools.translator; + +import org.junit.jupiter.api.Assumptions; +import org.junit.jupiter.api.DisplayName; +import org.junit.jupiter.api.Test; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.TimeUnit; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertTrue; +import static org.junit.jupiter.api.Assertions.fail; + +/** + * Mail, on both runtimes. + * + * The mail support is Java over the backend's own sockets and TLS, which are natives with two + * implementations, and its encoders run on two class libraries. The unit tests in + * maven/backend exercise the JVM half alone, so a STARTTLS upgrade that only works on the + * JDK's sockets, or a Base64 line that wraps differently on the translated class library, + * would pass all of them and ship. + * + * vm/backend/demo/mailcheck is self-contained: it starts an SMTP server of its own inside the + * process, sends to it through the real transport, and compares the bytes that arrived. It + * also runs the outbox against a database file and the security flows that end in a mail. + * This requires the translated binary and the JVM to pass the same number of checks. + */ +class BackendMailTest { + + @Test + @DisplayName("mail is encoded, sent, retried and captured the same on both runtimes") + void bothRuntimesAgree() throws Exception { + if (CompilerHelper.isWindows()) { + Assumptions.abort("the server-side backend is POSIX-only for now"); + } + BackendTestSupport.require(Files.isDirectory(BackendTestSupport.backendDir()), + "vm/backend is not present"); + Path jdk8 = BackendTestSupport.findJdk8(); + BackendTestSupport.require(jdk8 != null, "no JDK 8 available to build the backend"); + + Path work = Files.createTempDirectory("backend-mailcheck"); + Path binary = work.resolve("mailcheck"); + // Standalone: the check uses neither the shared contract nor demo/common. + String failure = BackendTestSupport.build("MailCheck", "demo/mailcheck", binary, jdk8, + true); + if (failure != null) { + BackendTestSupport.skipOrFail(failure); + } + + String translated = runTranslated(binary, work); + assertOk("the translated runtime", translated); + String local = runLocal(jdk8); + assertOk("the local Java SE runtime", local); + assertEquals(passedCount(local), passedCount(translated), + "the two runtimes ran a different number of checks\n--- translated ---\n" + + translated + "\n--- local ---\n" + local); + } + + private static String runTranslated(Path binary, Path work) throws Exception { + ProcessBuilder run = new ProcessBuilder(binary.toString()); + // The check keeps its database file in the working directory. + run.directory(work.toFile()); + run.redirectErrorStream(true); + Process p = run.start(); + boolean[] timedOut = new boolean[1]; + String output = BackendTestSupport.awaitOutput(p, 5, TimeUnit.MINUTES, timedOut); + if (timedOut[0]) { + fail("the translated mailcheck did not finish:\n" + output); + } + return output; + } + + private static String runLocal(Path jdk8) throws Exception { + Map env = new HashMap(); + env.put("CN1_BACKEND_JAVA", System.getProperty("java.home")); + env.put("CN1_BACKEND_DEMO", "demo/mailcheck"); + env.put("CN1_BACKEND_JDBC_JARS", BackendTestSupport.jdbcJars()); + env.put("JDK_8_HOME", jdk8.toString()); + int[] status = new int[1]; + return BackendTestSupport.runBackendScript( + new ArrayList(Arrays.asList("./run-javase.sh", "com.demo.MailCheck")), + env, 600, status); + } + + private static void assertOk(String which, String output) { + assertTrue(output.indexOf("MAILCHECK OK") >= 0, which + " failed:\n" + output); + // A floor, so a check that stopped checking cannot agree with itself about nothing. + assertTrue(passedCount(output) >= 200, + which + " ran only " + passedCount(output) + " checks:\n" + output); + } + + private static int passedCount(String output) { + int at = output.indexOf("passed="); + if (at < 0) { + return -1; + } + int end = output.indexOf(' ', at); + try { + return Integer.parseInt(output.substring(at + "passed=".length(), + end < 0 ? output.length() : end).trim()); + } catch (NumberFormatException err) { + return -1; + } + } +}