View Javadoc
1   /*
2    * DavMail POP/IMAP/SMTP/CalDav/LDAP Exchange Gateway
3    * Copyright (C) 2009  Mickael Guessant
4    *
5    * This program is free software; you can redistribute it and/or
6    * modify it under the terms of the GNU General Public License
7    * as published by the Free Software Foundation; either version 2
8    * of the License, or (at your option) any later version.
9    *
10   * This program is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU General Public License for more details.
14   *
15   * You should have received a copy of the GNU General Public License
16   * along with this program; if not, write to the Free Software
17   * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
18   */
19  package davmail.imap;
20  
21  import com.sun.mail.imap.protocol.BASE64MailboxDecoder;
22  import com.sun.mail.imap.protocol.BASE64MailboxEncoder;
23  import davmail.AbstractConnection;
24  import davmail.BundleMessage;
25  import davmail.DavGateway;
26  import davmail.Settings;
27  import davmail.exception.DavMailException;
28  import davmail.exception.HttpForbiddenException;
29  import davmail.exception.HttpNotFoundException;
30  import davmail.exception.InsufficientStorageException;
31  import davmail.exchange.ExchangeSession;
32  import davmail.exchange.ExchangeSessionFactory;
33  import davmail.exchange.MessageCreateThread;
34  import davmail.exchange.MessageLoadThread;
35  import davmail.ui.tray.DavGatewayTray;
36  import davmail.util.IOUtil;
37  import davmail.util.StringUtil;
38  import org.apache.http.client.HttpResponseException;
39  import org.apache.log4j.Logger;
40  
41  import javax.mail.MessagingException;
42  import javax.mail.internet.AddressException;
43  import javax.mail.internet.InternetAddress;
44  import javax.mail.internet.MimeBodyPart;
45  import javax.mail.internet.MimeMessage;
46  import javax.mail.internet.MimeMultipart;
47  import javax.mail.internet.MimePart;
48  import javax.mail.internet.MimeUtility;
49  import javax.mail.util.SharedByteArrayInputStream;
50  import java.io.ByteArrayOutputStream;
51  import java.io.FilterOutputStream;
52  import java.io.IOException;
53  import java.io.InputStream;
54  import java.io.OutputStream;
55  import java.io.UnsupportedEncodingException;
56  import java.net.Socket;
57  import java.net.SocketException;
58  import java.net.SocketTimeoutException;
59  import java.nio.charset.StandardCharsets;
60  import java.text.ParseException;
61  import java.text.SimpleDateFormat;
62  import java.util.ArrayList;
63  import java.util.Calendar;
64  import java.util.Date;
65  import java.util.Enumeration;
66  import java.util.HashMap;
67  import java.util.HashSet;
68  import java.util.Iterator;
69  import java.util.List;
70  import java.util.Locale;
71  import java.util.NoSuchElementException;
72  import java.util.Stack;
73  import java.util.TreeMap;
74  import java.util.UUID;
75  import java.util.concurrent.Executors;
76  import java.util.concurrent.ScheduledExecutorService;
77  import java.util.concurrent.ScheduledFuture;
78  import java.util.concurrent.TimeUnit;
79  
80  /**
81   * Dav Gateway IMAP connection implementation.
82   */
83  public class ImapConnection extends AbstractConnection {
84      private static final Logger LOGGER = Logger.getLogger(ImapConnection.class);
85  
86      protected String baseMailboxPath;
87      ExchangeSession.Folder currentFolder;
88  
89      /**
90       * Initialize the streams and start the thread.
91       *
92       * @param clientSocket IMAP client socket
93       */
94      public ImapConnection(Socket clientSocket) {
95          super(ImapConnection.class.getSimpleName(), clientSocket, "UTF-8");
96      }
97  
98      @Override
99      public void run() {
100         final String capabilities;
101         int imapIdleDelay = Settings.getIntProperty("davmail.imapIdleDelay") * 60;
102         if (imapIdleDelay > 0) {
103             capabilities = "CAPABILITY IMAP4REV1 AUTH=LOGIN IDLE MOVE SPECIAL-USE UIDPLUS";
104         } else {
105             capabilities = "CAPABILITY IMAP4REV1 AUTH=LOGIN MOVE SPECIAL-USE UIDPLUS";
106         }
107 
108         String line;
109         String commandId = null;
110         ImapTokenizer tokens;
111         try {
112             ExchangeSessionFactory.checkConfig();
113             sendClient("* OK [" + capabilities + "] IMAP4rev1 DavMail " + DavGateway.getCurrentVersion() + " server ready");
114             for (; ; ) {
115                 line = readClient();
116                 // unable to read line, connection closed ?
117                 if (line == null) {
118                     break;
119                 }
120 
121                 tokens = new ImapTokenizer(line);
122                 if (tokens.hasMoreTokens()) {
123                     commandId = tokens.nextToken();
124 
125                     checkInfiniteLoop(line);
126 
127                     if (tokens.hasMoreTokens()) {
128                         String command = tokens.nextToken();
129 
130                         if ("LOGOUT".equalsIgnoreCase(command)) {
131                             sendClient("* BYE Closing connection");
132                             sendClient(commandId + " OK LOGOUT completed");
133                             break;
134                         }
135                         if ("capability".equalsIgnoreCase(command)) {
136                             sendClient("* " + capabilities);
137                             sendClient(commandId + " OK CAPABILITY completed");
138                         } else if ("login".equalsIgnoreCase(command)) {
139                             parseCredentials(tokens);
140                             // detect shared mailbox access
141                             splitUserName();
142                             try {
143                                 session = ExchangeSessionFactory.getInstance(userName, password);
144                                 logConnection("LOGON", userName);
145                                 sendClient(commandId + " OK Authenticated");
146                                 state = State.AUTHENTICATED;
147                             } catch (Exception e) {
148                                 logConnection("FAILED", userName);
149                                 DavGatewayTray.error(e);
150                                 if (Settings.getBooleanProperty("davmail.enableKerberos")) {
151                                     sendClient(commandId + " NO LOGIN Kerberos authentication failed");
152                                 } else {
153                                     sendClient(commandId + " NO LOGIN failed");
154                                 }
155                                 state = State.INITIAL;
156                             }
157                         } else if ("AUTHENTICATE".equalsIgnoreCase(command)) {
158                             if (tokens.hasMoreTokens()) {
159                                 String authenticationMethod = tokens.nextToken();
160                                 if ("LOGIN".equalsIgnoreCase(authenticationMethod)) {
161                                     try {
162                                         sendClient("+ " + IOUtil.encodeBase64AsString("Username:"));
163                                         state = State.LOGIN;
164                                         userName = IOUtil.decodeBase64AsString(readClient());
165                                         // detect shared mailbox access
166                                         splitUserName();
167                                         sendClient("+ " + IOUtil.encodeBase64AsString("Password:"));
168                                         state = State.PASSWORD;
169                                         password = IOUtil.decodeBase64AsString(readClient());
170                                         session = ExchangeSessionFactory.getInstance(userName, password);
171                                         logConnection("LOGON", userName);
172                                         sendClient(commandId + " OK Authenticated");
173                                         state = State.AUTHENTICATED;
174                                     } catch (Exception e) {
175                                         logConnection("FAILED", userName);
176                                         DavGatewayTray.error(e);
177                                         sendClient(commandId + " NO LOGIN failed");
178                                         state = State.INITIAL;
179                                     }
180                                 } else {
181                                     sendClient(commandId + " NO unsupported authentication method");
182                                 }
183                             } else {
184                                 sendClient(commandId + " BAD authentication method required");
185                             }
186                         } else {
187                             if (state != State.AUTHENTICATED) {
188                                 sendClient(commandId + " BAD command authentication required");
189                             } else {
190                                 // check for expired session
191                                 session = ExchangeSessionFactory.getInstance(session, userName, password);
192                                 if ("lsub".equalsIgnoreCase(command) || "list".equalsIgnoreCase(command)) {
193                                     if (tokens.hasMoreTokens()) {
194                                         String token = tokens.nextToken();
195                                         // handle list-extended selection option
196                                         // see https://www.rfc-editor.org/rfc/rfc6154 5.2
197                                         boolean specialOnly = token.contains("SPECIAL-USE");
198                                         if (specialOnly && tokens.hasMoreTokens()) {
199                                             token = tokens.nextToken();
200                                         }
201 
202                                         if (tokens.hasMoreTokens()) {
203                                             String folderQuery = buildFolderPath(token) + decodeFolderPath(tokens.nextToken());
204                                             if (folderQuery.endsWith("%/%") && !"/%/%".equals(folderQuery)) {
205                                                 List<ExchangeSession.Folder> folders = session.getSubFolders(folderQuery.substring(0, folderQuery.length() - 3), false, false);
206                                                 for (ExchangeSession.Folder folder : folders) {
207                                                     sendClient("* " + command + " (" + folder.getFlags() + ") \"/\" \"" + encodeFolderPath(folder.folderPath) + '\"');
208                                                     sendSubFolders(command, folder.folderPath, false, false, specialOnly);
209                                                 }
210                                                 sendClient(commandId + " OK " + command + " completed");
211                                             } else if (folderQuery.endsWith("%") || folderQuery.endsWith("*")) {
212                                                 if ("/*".equals(folderQuery) || "/%".equals(folderQuery) || "/%/%".equals(folderQuery)) {
213                                                     folderQuery = folderQuery.substring(1);
214                                                     if ("%/%".equals(folderQuery)) {
215                                                         folderQuery = folderQuery.substring(0, folderQuery.length() - 2);
216                                                     }
217                                                     sendClient("* " + command + " (\\HasChildren) \"/\" \"/public\"");
218                                                 }
219                                                 if ("*%".equals(folderQuery)) {
220                                                     folderQuery = "*";
221                                                 }
222                                                 boolean wildcard = folderQuery.endsWith("%") && !folderQuery.contains("/") && !folderQuery.equals("%");
223                                                 boolean recursive = folderQuery.endsWith("*");
224                                                 sendSubFolders(command, folderQuery.substring(0, folderQuery.length() - 1), recursive, wildcard,
225                                                         specialOnly);
226                                                 sendClient(commandId + " OK " + command + " completed");
227                                             } else {
228                                                 ExchangeSession.Folder folder = null;
229                                                 try {
230                                                     folder = session.getFolder(folderQuery);
231                                                 } catch (HttpForbiddenException e) {
232                                                     // access forbidden, ignore
233                                                     DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_ACCESS_FORBIDDEN", folderQuery));
234                                                 } catch (HttpNotFoundException e) {
235                                                     // not found, ignore
236                                                     DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_NOT_FOUND", folderQuery));
237                                                 } catch (HttpResponseException e) {
238                                                     // other errors, ignore
239                                                     DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_ACCESS_ERROR", folderQuery, e.getMessage()));
240                                                 }
241                                                 if (folder != null) {
242                                                     sendClient("* " + command + " (" + folder.getFlags() + ") \"/\" \"" + encodeFolderPath(folder.folderPath) + '\"');
243                                                     sendClient(commandId + " OK " + command + " completed");
244                                                 } else {
245                                                     sendClient(commandId + " NO Folder not found");
246                                                 }
247                                             }
248                                         } else {
249                                             sendClient(commandId + " BAD missing folder argument");
250                                         }
251                                     } else {
252                                         sendClient(commandId + " BAD missing folder argument");
253                                     }
254                                 } else if ("select".equalsIgnoreCase(command) || "examine".equalsIgnoreCase(command)) {
255                                     if (tokens.hasMoreTokens()) {
256                                         @SuppressWarnings({"NonConstantStringShouldBeStringBuffer"})
257                                         String folderName = buildFolderPath(tokens.nextToken());
258                                         try {
259                                             currentFolder = session.getFolder(folderName);
260                                             loadFolder(currentFolder);
261                                             sendClient("* " + currentFolder.count() + " EXISTS");
262                                             sendClient("* " + currentFolder.recent + " RECENT");
263                                             sendClient("* OK [UIDVALIDITY 1]");
264                                             if (currentFolder.count() == 0) {
265                                                 sendClient("* OK [UIDNEXT 1]");
266                                             } else {
267                                                 sendClient("* OK [UIDNEXT " + currentFolder.getUidNext() + ']');
268                                             }
269                                             sendClient("* FLAGS (\\Answered \\Deleted \\Draft \\Flagged \\Seen $Forwarded Junk)");
270                                             sendClient("* OK [PERMANENTFLAGS (\\Answered \\Deleted \\Draft \\Flagged \\Seen $Forwarded Junk \\*)]");
271                                             if ("select".equalsIgnoreCase(command)) {
272                                                 sendClient(commandId + " OK [READ-WRITE] " + command + " completed");
273                                             } else {
274                                                 sendClient(commandId + " OK [READ-ONLY] " + command + " completed");
275                                             }
276                                         } catch (HttpNotFoundException e) {
277                                             sendClient(commandId + " NO Not found");
278                                         } catch (HttpForbiddenException e) {
279                                             sendClient(commandId + " NO Forbidden");
280                                         }
281                                     } else {
282                                         sendClient(commandId + " BAD command unrecognized");
283                                     }
284                                 } else if ("expunge".equalsIgnoreCase(command)) {
285                                     if (expunge(false)) {
286                                         // need to refresh folder to avoid 404 errors
287                                         session.refreshFolder(currentFolder);
288                                     }
289                                     sendClient(commandId + " OK " + command + " completed");
290                                 } else if ("close".equalsIgnoreCase(command)) {
291                                     expunge(true);
292                                     // deselect folder
293                                     currentFolder = null;
294                                     sendClient(commandId + " OK " + command + " completed");
295                                 } else if ("create".equalsIgnoreCase(command)) {
296                                     if (tokens.hasMoreTokens()) {
297                                         session.createMessageFolder(buildFolderPath(tokens.nextToken()));
298                                         sendClient(commandId + " OK folder created");
299                                     } else {
300                                         sendClient(commandId + " BAD missing create argument");
301                                     }
302                                 } else if ("rename".equalsIgnoreCase(command)) {
303                                     String folderName = buildFolderPath(tokens.nextToken());
304                                     String targetName = buildFolderPath(tokens.nextToken());
305                                     try {
306                                         session.moveFolder(folderName, targetName);
307                                         sendClient(commandId + " OK rename completed");
308                                     } catch (HttpResponseException e) {
309                                         sendClient(commandId + " NO " + e.getMessage());
310                                     }
311                                 } else if ("delete".equalsIgnoreCase(command)) {
312                                     String folderName = buildFolderPath(tokens.nextToken());
313                                     try {
314                                         session.deleteFolder(folderName);
315                                         sendClient(commandId + " OK folder deleted");
316                                     } catch (HttpResponseException e) {
317                                         sendClient(commandId + " NO " + e.getMessage());
318                                     }
319                                 } else if ("uid".equalsIgnoreCase(command)) {
320                                     if (tokens.hasMoreTokens()) {
321                                         String subcommand = tokens.nextToken();
322                                         if ("fetch".equalsIgnoreCase(subcommand)) {
323                                             if (currentFolder == null) {
324                                                 sendClient(commandId + " NO no folder selected");
325                                             } else {
326                                                 String ranges = tokens.nextToken();
327                                                 if (ranges == null) {
328                                                     sendClient(commandId + " BAD missing range parameter");
329                                                 } else {
330                                                     String parameters = null;
331                                                     if (tokens.hasMoreTokens()) {
332                                                         parameters = tokens.nextToken();
333                                                     }
334                                                     UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, ranges);
335                                                     while (uidRangeIterator.hasNext()) {
336                                                         DavGatewayTray.switchIcon();
337                                                         ExchangeSession.Message message = uidRangeIterator.next();
338                                                         try {
339                                                             handleFetch(message, uidRangeIterator.currentIndex, parameters);
340                                                         } catch (HttpNotFoundException e) {
341                                                             LOGGER.warn("Ignore missing message " + uidRangeIterator.currentIndex);
342                                                         } catch (SocketException e) {
343                                                             // client closed connection
344                                                             throw e;
345                                                         } catch (IOException e) {
346                                                             DavGatewayTray.log(e);
347                                                             LOGGER.warn("Ignore broken message " + uidRangeIterator.currentIndex + ' ' + e.getMessage());
348                                                         }
349                                                     }
350                                                     sendClient(commandId + " OK UID FETCH completed");
351                                                 }
352                                             }
353 
354                                         } else if ("search".equalsIgnoreCase(subcommand)) {
355                                             List<Long> uidList = handleSearch(tokens);
356                                             StringBuilder buffer = new StringBuilder("* SEARCH");
357                                             for (long uid : uidList) {
358                                                 buffer.append(' ');
359                                                 buffer.append(uid);
360                                             }
361                                             sendClient(buffer.toString());
362                                             sendClient(commandId + " OK SEARCH completed");
363 
364                                         } else if ("store".equalsIgnoreCase(subcommand)) {
365                                             UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, tokens.nextToken());
366                                             String action = tokens.nextToken();
367                                             String flags = tokens.nextToken();
368                                             handleStore(commandId, uidRangeIterator, action, flags);
369                                         } else if ("copy".equalsIgnoreCase(subcommand) || "move".equalsIgnoreCase(subcommand)) {
370                                             try {
371                                                 UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, tokens.nextToken());
372                                                 String targetName = buildFolderPath(tokens.nextToken());
373                                                 if (!uidRangeIterator.hasNext()) {
374                                                     sendClient(commandId + " NO " + "No message found");
375                                                 } else {
376                                                     ArrayList<ExchangeSession.Message> messages = new ArrayList<>();
377                                                     while (uidRangeIterator.hasNext()) {
378                                                         messages.add(uidRangeIterator.next());
379                                                     }
380                                                     if ("copy".equalsIgnoreCase(subcommand)) {
381                                                         session.copyMessages(messages, targetName);
382                                                     } else {
383                                                         session.moveMessages(messages, targetName);
384                                                     }
385                                                     sendClient(commandId + " OK " + subcommand + " completed");
386                                                 }
387                                             } catch (HttpNotFoundException e) {
388                                                 sendClient(commandId + " NO [TRYCREATE] " + e.getMessage());
389                                             } catch (HttpResponseException e) {
390                                                 sendClient(commandId + " NO " + e.getMessage());
391                                             }
392                                         }
393                                     } else {
394                                         sendClient(commandId + " BAD command unrecognized");
395                                     }
396                                 } else if ("search".equalsIgnoreCase(command)) {
397                                     if (currentFolder == null) {
398                                         sendClient(commandId + " NO no folder selected");
399                                     } else {
400                                         List<Long> uidList = handleSearch(tokens);
401                                         if (uidList.isEmpty()) {
402                                             sendClient("* SEARCH");
403                                         } else {
404                                             int currentIndex = 0;
405                                             StringBuilder buffer = new StringBuilder("* SEARCH");
406                                             for (ExchangeSession.Message message : currentFolder.messages) {
407                                                 currentIndex++;
408                                                 if (uidList.contains(message.getImapUid())) {
409                                                     buffer.append(' ');
410                                                     buffer.append(currentIndex);
411                                                 }
412                                             }
413                                             sendClient(buffer.toString());
414                                         }
415                                         sendClient(commandId + " OK SEARCH completed");
416                                     }
417                                 } else if ("fetch".equalsIgnoreCase(command)) {
418                                     if (currentFolder == null) {
419                                         sendClient(commandId + " NO no folder selected");
420                                     } else {
421                                         RangeIterator rangeIterator = new RangeIterator(currentFolder.messages, tokens.nextToken());
422                                         String parameters = null;
423                                         if (tokens.hasMoreTokens()) {
424                                             parameters = tokens.nextToken();
425                                         }
426                                         while (rangeIterator.hasNext()) {
427                                             DavGatewayTray.switchIcon();
428                                             ExchangeSession.Message message = rangeIterator.next();
429                                             try {
430                                                 handleFetch(message, rangeIterator.currentIndex, parameters);
431                                             } catch (HttpNotFoundException e) {
432                                                 LOGGER.warn("Ignore missing message " + rangeIterator.currentIndex);
433                                             } catch (SocketException e) {
434                                                 // client closed connection, rethrow exception
435                                                 throw e;
436                                             } catch (IOException e) {
437                                                 DavGatewayTray.log(e);
438                                                 LOGGER.warn("Ignore broken message " + rangeIterator.currentIndex + ' ' + e.getMessage());
439                                             }
440 
441                                         }
442                                         sendClient(commandId + " OK FETCH completed");
443                                     }
444 
445                                 } else if ("store".equalsIgnoreCase(command)) {
446                                     RangeIterator rangeIterator = new RangeIterator(currentFolder.messages, tokens.nextToken());
447                                     String action = tokens.nextToken();
448                                     String flags = tokens.nextToken();
449                                     handleStore(commandId, rangeIterator, action, flags);
450 
451                                 } else if ("copy".equalsIgnoreCase(command) || "move".equalsIgnoreCase(command)) {
452                                     try {
453                                         RangeIterator rangeIterator = new RangeIterator(currentFolder.messages, tokens.nextToken());
454                                         String targetName = buildFolderPath(tokens.nextToken());
455                                         if (!rangeIterator.hasNext()) {
456                                             sendClient(commandId + " NO " + "No message found");
457                                         } else {
458                                             while (rangeIterator.hasNext()) {
459                                                 DavGatewayTray.switchIcon();
460                                                 ExchangeSession.Message message = rangeIterator.next();
461                                                 if ("copy".equalsIgnoreCase(command)) {
462                                                     session.copyMessage(message, targetName);
463                                                 } else {
464                                                     session.moveMessage(message, targetName);
465                                                 }
466                                             }
467                                             sendClient(commandId + " OK " + command + " completed");
468                                         }
469                                     } catch (HttpResponseException e) {
470                                         sendClient(commandId + " NO " + e.getMessage());
471                                     }
472                                 } else if ("append".equalsIgnoreCase(command)) {
473                                     String folderName = buildFolderPath(tokens.nextToken());
474                                     HashMap<String, String> properties = new HashMap<>();
475                                     String flags = null;
476                                     String date = null;
477                                     // handle optional flags
478                                     String nextToken = tokens.nextQuotedToken();
479                                     if (nextToken.startsWith("(")) {
480                                         flags = StringUtil.removeQuotes(nextToken);
481                                         if (tokens.hasMoreTokens()) {
482                                             nextToken = tokens.nextToken();
483                                             if (tokens.hasMoreTokens()) {
484                                                 date = nextToken;
485                                                 nextToken = tokens.nextToken();
486                                             }
487                                         }
488                                     } else if (tokens.hasMoreTokens()) {
489                                         date = StringUtil.removeQuotes(nextToken);
490                                         nextToken = tokens.nextToken();
491                                     }
492 
493                                     if (flags != null) {
494                                         HashSet<String> keywords = null;
495                                         // parse flags, on create read and draft flags are on the
496                                         // same messageFlags property, 8 means draft and 1 means read
497                                         ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
498                                         while (flagtokenizer.hasMoreTokens()) {
499                                             String flag = flagtokenizer.nextToken();
500                                             if ("\\Seen".equalsIgnoreCase(flag)) {
501                                                 if (properties.containsKey("draft")) {
502                                                     // draft message, add read flag
503                                                     properties.put("draft", "9");
504                                                 } else {
505                                                     // not (yet) draft, set read flag
506                                                     properties.put("draft", "1");
507                                                 }
508                                             } else if ("\\Flagged".equalsIgnoreCase(flag)) {
509                                                 properties.put("flagged", "2");
510                                             } else if ("\\Answered".equalsIgnoreCase(flag)) {
511                                                 properties.put("answered", "102");
512                                             } else if ("$Forwarded".equalsIgnoreCase(flag)) {
513                                                 properties.put("forwarded", "104");
514                                             } else if ("\\Draft".equalsIgnoreCase(flag)) {
515                                                 if (properties.containsKey("draft")) {
516                                                     // read message, add draft flag
517                                                     properties.put("draft", "9");
518                                                 } else {
519                                                     // not (yet) read, set draft flag
520                                                     properties.put("draft", "8");
521                                                 }
522                                             } else if ("Junk".equalsIgnoreCase(flag)) {
523                                                 properties.put("junk", "1");
524                                             } else {
525                                                 if (keywords == null) {
526                                                     keywords = new HashSet<>();
527                                                 }
528                                                 keywords.add(flag);
529                                             }
530                                         }
531                                         if (keywords != null) {
532                                             properties.put("keywords", session.convertFlagsToKeywords(keywords));
533                                         }
534                                     } else {
535                                         // no flags, force not draft and unread
536                                         properties.put("draft", "0");
537                                     }
538                                     // handle optional date
539                                     if (date != null) {
540                                         SimpleDateFormat dateParser = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss Z", Locale.ENGLISH);
541                                         Date dateReceived = dateParser.parse(date);
542                                         SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
543                                         dateFormatter.setTimeZone(ExchangeSession.GMT_TIMEZONE);
544 
545                                         properties.put("datereceived", dateFormatter.format(dateReceived));
546                                     }
547                                     int size = Integer.parseInt(StringUtil.removeQuotes(nextToken));
548                                     sendClient("+ send literal data");
549                                     byte[] buffer = in.readContent(size);
550                                     // empty line
551                                     readClient();
552                                     MimeMessage mimeMessage = new MimeMessage(null, new SharedByteArrayInputStream(buffer));
553 
554                                     String messageName = UUID.randomUUID() + ".EML";
555                                     try {
556                                         ExchangeSession.Message createdMessage = MessageCreateThread.createMessage(session, folderName, messageName, properties, mimeMessage, os, capabilities);
557                                         if (createdMessage != null) {
558                                             long uid = createdMessage.getImapUid();
559                                             sendClient(commandId + " OK [APPENDUID 1 "+uid+"] APPEND completed");
560                                         } else {
561                                             sendClient(commandId + " OK APPEND completed");
562                                         }
563                                     } catch (InsufficientStorageException e) {
564                                         sendClient(commandId + " NO " + e.getMessage());
565                                     }
566                                 } else if ("idle".equalsIgnoreCase(command) && imapIdleDelay > 0) {
567                                     if (currentFolder != null) {
568                                         sendClient("+ idling ");
569                                         // clear cache before going to idle mode
570                                         currentFolder.clearCache();
571                                         DavGatewayTray.resetIcon();
572                                         int originalTimeout = client.getSoTimeout();
573                                         try {
574                                             int count = 0;
575                                             client.setSoTimeout(1000);
576                                             while (in.available() == 0) {
577                                                 if (++count >= imapIdleDelay) {
578                                                     count = 0;
579                                                     TreeMap<Long, String> previousImapFlagMap = currentFolder.getImapFlagMap();
580                                                     if (session.refreshFolder(currentFolder)) {
581                                                         handleRefresh(previousImapFlagMap, currentFolder.getImapFlagMap());
582                                                     }
583                                                 }
584                                                 // wait for input 1 second
585                                                 try {
586                                                     byte[] byteBuffer = new byte[1];
587                                                     if (in.read(byteBuffer) > 0) {
588                                                         in.unread(byteBuffer);
589                                                     }
590                                                 } catch (SocketTimeoutException e) {
591                                                     // ignore, read timed out
592                                                 }
593                                             }
594                                             // read DONE line
595                                             line = readClient();
596                                             if ("DONE".equals(line)) {
597                                                 sendClient(commandId + " OK " + command + " terminated");
598                                             } else {
599                                                 sendClient(commandId + " BAD command unrecognized");
600                                             }
601                                         } catch (IOException e) {
602                                             // client connection closed
603                                             throw new SocketException(e.getMessage());
604                                         } finally {
605                                             client.setSoTimeout(originalTimeout);
606                                         }
607                                     } else {
608                                         sendClient(commandId + " NO no folder selected");
609                                     }
610                                 } else if ("noop".equalsIgnoreCase(command) || "check".equalsIgnoreCase(command)) {
611                                     if (currentFolder != null) {
612                                         DavGatewayTray.debug(new BundleMessage("LOG_IMAP_COMMAND", command, currentFolder.folderPath));
613                                         TreeMap<Long, String> previousImapFlagMap = currentFolder.getImapFlagMap();
614                                         if (session.refreshFolder(currentFolder)) {
615                                             handleRefresh(previousImapFlagMap, currentFolder.getImapFlagMap());
616                                         }
617                                     }
618                                     sendClient(commandId + " OK " + command + " completed");
619                                 } else if ("subscribe".equalsIgnoreCase(command) || "unsubscribe".equalsIgnoreCase(command)) {
620                                     sendClient(commandId + " OK " + command + " completed");
621                                 } else if ("status".equalsIgnoreCase(command)) {
622                                     try {
623                                         String encodedFolderName = tokens.nextToken();
624                                         ExchangeSession.Folder folder = session.getFolder(buildFolderPath(encodedFolderName));
625                                         // must retrieve messages
626 
627                                         loadFolder(folder);
628 
629                                         String parameters = tokens.nextToken();
630                                         StringBuilder answer = new StringBuilder();
631                                         ImapTokenizer parametersTokens = new ImapTokenizer(parameters);
632                                         while (parametersTokens.hasMoreTokens()) {
633                                             String token = parametersTokens.nextToken();
634                                             if ("MESSAGES".equalsIgnoreCase(token)) {
635                                                 answer.append("MESSAGES ").append(folder.count()).append(' ');
636                                             }
637                                             if ("RECENT".equalsIgnoreCase(token)) {
638                                                 answer.append("RECENT ").append(folder.recent).append(' ');
639                                             }
640                                             if ("UIDNEXT".equalsIgnoreCase(token)) {
641                                                 if (folder.count() == 0) {
642                                                     answer.append("UIDNEXT 1 ");
643                                                 } else {
644                                                     if (folder.count() == 0) {
645                                                         answer.append("UIDNEXT 1 ");
646                                                     } else {
647                                                         answer.append("UIDNEXT ").append(folder.getUidNext()).append(' ');
648                                                     }
649                                                 }
650 
651                                             }
652                                             if ("UIDVALIDITY".equalsIgnoreCase(token)) {
653                                                 answer.append("UIDVALIDITY 1 ");
654                                             }
655                                             if ("UNSEEN".equalsIgnoreCase(token)) {
656                                                 answer.append("UNSEEN ").append(folder.unreadCount).append(' ');
657                                             }
658                                         }
659                                         sendClient("* STATUS \"" + encodedFolderName + "\" (" + answer.toString().trim() + ')');
660                                         sendClient(commandId + " OK " + command + " completed");
661                                     } catch (HttpResponseException e) {
662                                         sendClient(commandId + " NO folder not found");
663                                     }
664                                 } else {
665                                     sendClient(commandId + " BAD command unrecognized");
666                                 }
667                             }
668                         }
669 
670                     } else {
671                         sendClient(commandId + " BAD missing command");
672                     }
673                 } else {
674                     sendClient("BAD Null command");
675                 }
676                 DavGatewayTray.resetIcon();
677             }
678 
679             os.flush();
680         } catch (SocketTimeoutException e) {
681             DavGatewayTray.debug(new BundleMessage("LOG_CLOSE_CONNECTION_ON_TIMEOUT"));
682             try {
683                 sendClient("* BYE Closing connection");
684             } catch (IOException e1) {
685                 DavGatewayTray.debug(new BundleMessage("LOG_EXCEPTION_CLOSING_CONNECTION_ON_TIMEOUT"));
686             }
687         } catch (SocketException e) {
688             LOGGER.warn(BundleMessage.formatLog("LOG_CLIENT_CLOSED_CONNECTION"));
689         } catch (Exception e) {
690             DavGatewayTray.log(e);
691             try {
692                 String message = ((e.getMessage() == null) ? e.toString() : e.getMessage()).replaceAll("\\n", " ");
693                 if (commandId != null) {
694                     sendClient(commandId + " BAD unable to handle request: " + message);
695                 } else {
696                     sendClient("* BAD unable to handle request: " + message);
697                 }
698             } catch (IOException e2) {
699                 DavGatewayTray.warn(new BundleMessage("LOG_EXCEPTION_SENDING_ERROR_TO_CLIENT"), e2);
700             }
701         } finally {
702             close();
703         }
704         DavGatewayTray.resetIcon();
705     }
706 
707     protected String lastCommand;
708     protected int lastCommandCount;
709 
710     /**
711      * Detect infinite loop on the client side.
712      *
713      * @param line IMAP command line
714      * @throws IOException on infinite loop
715      */
716     protected void checkInfiniteLoop(String line) throws IOException {
717         int spaceIndex = line.indexOf(' ');
718         if (spaceIndex < 0) {
719             // invalid command line, reset
720             lastCommand = null;
721             lastCommandCount = 0;
722         } else {
723             String command = line.substring(spaceIndex + 1);
724             if (command.equals(lastCommand)) {
725                 lastCommandCount++;
726                 if (lastCommandCount > 100 && !"NOOP".equalsIgnoreCase(lastCommand) && !"IDLE".equalsIgnoreCase(lastCommand)) {
727                     // more than a hundred times the same command => this is a client infinite loop, close connection
728                     throw new IOException("Infinite loop on command " + command + " detected");
729                 }
730             } else {
731                 // new command, reset
732                 lastCommand = command;
733                 lastCommandCount = 0;
734             }
735         }
736     }
737 
738     /**
739      * Detect shared mailbox access.
740      * see <a href="https://help.ubuntu.com/community/ThunderbirdExchange">Connecting to a Microsoft Exchange Server with Thunderbird</a>
741      */
742     protected void splitUserName() {
743         String[] tokens = null;
744         if (userName.indexOf('/') >= 0) {
745             tokens = userName.split("/");
746         } else if (userName.indexOf('\\') >= 0) {
747             tokens = userName.split("\\\\");
748         }
749 
750         if (tokens != null && tokens.length == 3) {
751             userName = tokens[0] + '\\' + tokens[1];
752             baseMailboxPath = "/users/" + tokens[2] + '/';
753         } else if (tokens != null && tokens.length == 2 && tokens[1].contains("@")) {
754             userName = tokens[0];
755             baseMailboxPath = "/users/" + tokens[1] + '/';
756         }
757     }
758 
759     protected String encodeFolderPath(String folderPath) {
760         return BASE64MailboxEncoder.encode(folderPath).replaceAll("\"", "\\\\\"");
761     }
762 
763     protected String decodeFolderPath(String folderPath) {
764         return BASE64MailboxDecoder.decode(folderPath)
765                 //unescape quotes inside value
766                 .replaceAll("\\\\", "");
767     }
768 
769     protected String buildFolderPath(String encodedFolderPath) {
770         String decodedFolderPath = decodeFolderPath(encodedFolderPath);
771         if (baseMailboxPath == null || decodedFolderPath.startsWith("/")) {
772             return decodedFolderPath;
773         } else {
774             return baseMailboxPath + decodedFolderPath;
775         }
776     }
777 
778     /**
779      * Send expunge untagged response for removed IMAP message uids.
780      *
781      * @param previousImapFlagMap uid map before refresh
782      * @param imapFlagMap         uid map after refresh
783      * @throws IOException on error
784      */
785     private void handleRefresh(TreeMap<Long, String> previousImapFlagMap, TreeMap<Long, String> imapFlagMap) throws IOException {
786         // send deleted message expunge notification
787         int index = 1;
788         for (long previousImapUid : previousImapFlagMap.keySet()) {
789             if (!imapFlagMap.containsKey(previousImapUid)) {
790                 sendClient("* " + index + " EXPUNGE");
791             } else {
792                 // send updated flags
793                 if (!previousImapFlagMap.get(previousImapUid).equals(imapFlagMap.get(previousImapUid))) {
794                     sendClient("* " + index + " FETCH (UID " + previousImapUid + " FLAGS (" + imapFlagMap.get(previousImapUid) + "))");
795                 }
796                 index++;
797             }
798         }
799 
800         sendClient("* " + currentFolder.count() + " EXISTS");
801         sendClient("* " + currentFolder.recent + " RECENT");
802     }
803 
804     static private class KeepAlive {
805         private ScheduledExecutorService scheduler;
806         private ScheduledFuture<?> task;
807 
808         public KeepAlive(Runnable cb) {
809             scheduler = Executors.newScheduledThreadPool(1, r -> new Thread(r, currentThread().getName() + "-KeepAlive"));
810             task = scheduler.scheduleAtFixedRate(cb, 20, 20, TimeUnit.SECONDS);
811         }
812 
813         public void cancel() {
814             task.cancel(true);
815             scheduler.shutdown();
816         }
817     }
818 
819     private void loadFolder(ExchangeSession.Folder folder) throws IOException {
820         final boolean enable = Settings.getBooleanProperty("davmail.enableKeepAlive", false);
821         KeepAlive keepAlive = null;
822 
823         if (enable && (folder.count() > 500)) {
824             Runnable progressNotifier = () -> {
825                 try {
826                     sendClient("* OK in progress");
827                 } catch (IOException e) {
828                     LOGGER.warn("Error while sending progress notification", e);
829                 }
830             };
831             keepAlive = new KeepAlive(progressNotifier);
832         }
833 
834         try {
835             folder.loadMessages();
836         } finally {
837             if (keepAlive != null) {
838                 keepAlive.cancel();
839             }
840         }
841     }
842 
843     static protected class MessageWrapper {
844         protected OutputStream os;
845         protected StringBuilder buffer;
846         protected ExchangeSession.Message message;
847 
848         protected MessageWrapper(OutputStream os, StringBuilder buffer, ExchangeSession.Message message) {
849             this.os = os;
850             this.buffer = buffer;
851             this.message = message;
852         }
853 
854         public int getMimeMessageSize() throws IOException, MessagingException {
855             loadMessage();
856             return message.getMimeMessageSize();
857         }
858 
859         /**
860          * Monitor full message download
861          */
862         protected void loadMessage() throws IOException, MessagingException {
863             if (!message.isLoaded()) {
864                 // flush current buffer
865                 String flushString = buffer.toString();
866                 LOGGER.debug(flushString);
867                 os.write(flushString.getBytes(StandardCharsets.UTF_8));
868                 buffer.setLength(0);
869                 MessageLoadThread.loadMimeMessage(message, os);
870             }
871         }
872 
873         public MimeMessage getMimeMessage() throws IOException, MessagingException {
874             loadMessage();
875             return message.getMimeMessage();
876         }
877 
878         public InputStream getRawInputStream() throws IOException, MessagingException {
879             loadMessage();
880             return message.getRawInputStream();
881         }
882 
883         public Enumeration getMatchingHeaderLines(String[] requestedHeaders) throws IOException, MessagingException {
884             Enumeration result = message.getMatchingHeaderLinesFromHeaders(requestedHeaders);
885             if (result == null) {
886                 loadMessage();
887                 result = message.getMatchingHeaderLines(requestedHeaders);
888             }
889             return result;
890         }
891     }
892 
893 
894     private void handleFetch(ExchangeSession.Message message, int currentIndex, String parameters) throws IOException, MessagingException {
895         StringBuilder buffer = new StringBuilder();
896         MessageWrapper messageWrapper = new MessageWrapper(os, buffer, message);
897         buffer.append("* ").append(currentIndex).append(" FETCH (UID ").append(message.getImapUid());
898         if (parameters != null) {
899             parameters = handleFetchMacro(parameters);
900             ImapTokenizer paramTokens = new ImapTokenizer(parameters);
901             while (paramTokens.hasMoreTokens()) {
902                 @SuppressWarnings({"NonConstantStringShouldBeStringBuffer"})
903                 String param = paramTokens.nextToken().toUpperCase();
904                 if ("FLAGS".equals(param)) {
905                     buffer.append(" FLAGS (").append(message.getImapFlags()).append(')');
906                 } else if ("RFC822.SIZE".equals(param)) {
907                     int size;
908                     if ((parameters.contains("BODY.PEEK[HEADER.FIELDS (")
909                             // exclude mutt header request
910                             && !parameters.contains("X-LABEL"))
911                             || parameters.equals("RFC822.SIZE RFC822.HEADER FLAGS") // icedove
912                             || Settings.getBooleanProperty("davmail.imapAlwaysApproxMsgSize")) {   // Send approximate size
913                         size = message.size;
914                         LOGGER.debug(String.format("Message %s sent approximate size %d bytes", message.getImapUid(), size));
915                     } else {
916                         size = messageWrapper.getMimeMessageSize();
917                     }
918                     buffer.append(" RFC822.SIZE ").append(size);
919                 } else if ("ENVELOPE".equals(param)) {
920                     appendEnvelope(buffer, messageWrapper);
921                 } else if ("BODYSTRUCTURE".equals(param)) {
922                     appendBodyStructure(buffer, messageWrapper);
923                 } else if ("INTERNALDATE".equals(param) && message.date != null && !message.date.isEmpty()) {
924                     try {
925                         SimpleDateFormat dateParser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
926                         dateParser.setTimeZone(ExchangeSession.GMT_TIMEZONE);
927                         Date date = ExchangeSession.getZuluDateFormat().parse(message.date);
928                         SimpleDateFormat dateFormatter = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss Z", Locale.ENGLISH);
929                         buffer.append(" INTERNALDATE \"").append(dateFormatter.format(date)).append('\"');
930                     } catch (ParseException e) {
931                         throw new DavMailException("EXCEPTION_INVALID_DATE", message.date);
932                     }
933                 } else if ("RFC822".equals(param) || param.startsWith("BODY[") || param.startsWith("BODY.PEEK[")
934                         || "RFC822.HEADER".equals(param) || "RFC822.TEXT".equals(param)) {
935 
936                     if (param.startsWith("BODY[") && !message.read) {
937                         // According to IMAP RFC: The \Seen flag is implicitly set
938                         updateFlags(message, "FLAGS", "\\Seen");
939                         message.read = true;
940                     }
941 
942                     // get full param
943                     if (param.indexOf('[') >= 0) {
944                         StringBuilder paramBuffer = new StringBuilder(param);
945                         while (paramTokens.hasMoreTokens() && paramBuffer.indexOf("]") < 0) {
946                             paramBuffer.append(' ').append(paramTokens.nextToken());
947                         }
948                         param = paramBuffer.toString();
949                     }
950                     // parse buffer size
951                     int startIndex = 0;
952                     int maxSize = Integer.MAX_VALUE;
953                     int ltIndex = param.indexOf('<');
954                     if (ltIndex >= 0) {
955                         int dotIndex = param.indexOf('.', ltIndex);
956                         if (dotIndex >= 0) {
957                             startIndex = Integer.parseInt(param.substring(ltIndex + 1, dotIndex));
958                             maxSize = Integer.parseInt(param.substring(dotIndex + 1, param.indexOf('>')));
959                         }
960                     }
961 
962                     ByteArrayOutputStream baos = new ByteArrayOutputStream();
963                     InputStream partInputStream = null;
964                     OutputStream partOutputStream = null;
965 
966                     // try to parse message part index
967                     String partIndexString = StringUtil.getToken(param, "[", "]");
968                     if ((partIndexString == null || partIndexString.isEmpty()) && !"RFC822.HEADER".equals(param)) {
969                         // write message with headers
970                         partOutputStream = new PartialOutputStream(baos, startIndex, maxSize);
971                         partInputStream = messageWrapper.getRawInputStream();
972                     } else if ("TEXT".equals(partIndexString)) {
973                         // write message without headers
974                         partOutputStream = new PartOutputStream(baos, false, true, startIndex, maxSize);
975                         partInputStream = messageWrapper.getRawInputStream();
976                     } else if ("RFC822.HEADER".equals(param) || (partIndexString != null && partIndexString.startsWith("HEADER"))) {
977                         // Header requested fetch  headers
978                         String[] requestedHeaders = getRequestedHeaders(partIndexString);
979                         // OSX Lion special flags request
980                         if (requestedHeaders != null && requestedHeaders.length == 1 && "content-class".equals(requestedHeaders[0]) && message.contentClass != null) {
981                             baos.write("Content-class: ".getBytes(StandardCharsets.UTF_8));
982                             baos.write(message.contentClass.getBytes(StandardCharsets.UTF_8));
983                             baos.write(13);
984                             baos.write(10);
985                         } else if (requestedHeaders == null) {
986                             // load message and write all headers
987                             partOutputStream = new PartOutputStream(baos, true, false, startIndex, maxSize);
988                             partInputStream = messageWrapper.getRawInputStream();
989                         } else {
990                             Enumeration headerEnumeration = messageWrapper.getMatchingHeaderLines(requestedHeaders);
991                             while (headerEnumeration.hasMoreElements()) {
992                                 baos.write(((String) headerEnumeration.nextElement()).getBytes(StandardCharsets.UTF_8));
993                                 baos.write(13);
994                                 baos.write(10);
995                             }
996                         }
997                     } else if (partIndexString != null) {
998                         MimePart bodyPart = messageWrapper.getMimeMessage();
999                         String[] partIndexStrings = partIndexString.split("\\.");
1000                         for (String subPartIndexString : partIndexStrings) {
1001                             // ignore MIME subpart index, will return full part
1002                             if ("MIME".equals(subPartIndexString)) {
1003                                 break;
1004                             }
1005                             int subPartIndex;
1006                             // try to parse part index
1007                             try {
1008                                 subPartIndex = Integer.parseInt(subPartIndexString);
1009                             } catch (NumberFormatException e) {
1010                                 throw new DavMailException("EXCEPTION_INVALID_PARAMETER", param);
1011                             }
1012 
1013                             Object mimeBody = bodyPart.getContent();
1014                             if (mimeBody instanceof MimeMultipart) {
1015                                 MimeMultipart multiPart = (MimeMultipart) mimeBody;
1016                                 if (subPartIndex - 1 < multiPart.getCount()) {
1017                                     bodyPart = (MimePart) multiPart.getBodyPart(subPartIndex - 1);
1018                                 } else {
1019                                     throw new DavMailException("EXCEPTION_INVALID_PARAMETER", param);
1020                                 }
1021                             } else if (subPartIndex != 1) {
1022                                 throw new DavMailException("EXCEPTION_INVALID_PARAMETER", param);
1023                             }
1024                         }
1025 
1026                         // write selected part, without headers
1027                         partOutputStream = new PartialOutputStream(baos, startIndex, maxSize);
1028                         if (bodyPart instanceof MimeMessage) {
1029                             partInputStream = ((MimeMessage) bodyPart).getRawInputStream();
1030                         } else {
1031                             partInputStream = ((MimeBodyPart) bodyPart).getRawInputStream();
1032                         }
1033                     }
1034 
1035                     // copy selected content to baos
1036                     if (partInputStream != null && partOutputStream != null) {
1037                         IOUtil.write(partInputStream, partOutputStream);
1038                         partInputStream.close();
1039                         partOutputStream.close();
1040                     }
1041                     baos.close();
1042 
1043                     if ("RFC822".equals(param)) {
1044                         buffer.append(" RFC822");
1045                     } else if ("RFC822.HEADER".equals(param)) {
1046                         buffer.append(" RFC822.HEADER");
1047                     } else if ("RFC822.TEXT".equals(param)) {
1048                         buffer.append(" RFC822.TEXT");
1049                     } else {
1050                         buffer.append(" BODY[");
1051                         if (partIndexString != null) {
1052                             buffer.append(partIndexString);
1053                         }
1054                         buffer.append(']');
1055                     }
1056                     // partial
1057                     if (startIndex > 0 || maxSize != Integer.MAX_VALUE) {
1058                         buffer.append('<').append(startIndex).append('>');
1059                     }
1060                     buffer.append(" {").append(baos.size()).append('}');
1061                     sendClient(buffer.toString());
1062                     // log content if less than 2K
1063                     if (LOGGER.isDebugEnabled() && baos.size() < 2048) {
1064                         LOGGER.debug(new String(baos.toByteArray(), StandardCharsets.UTF_8));
1065                     }
1066                     os.write(baos.toByteArray());
1067                     os.flush();
1068                     buffer.setLength(0);
1069                 }
1070             }
1071         }
1072         buffer.append(')');
1073         sendClient(buffer.toString());
1074         // do not keep message content in memory
1075         message.dropMimeMessage();
1076     }
1077 
1078     /**
1079      * Handle flags macro in fetch requests
1080      * @param parameters input fetch flags
1081      * @return transformed fetch flags
1082      */
1083     private String handleFetchMacro(String parameters) {
1084         if ("ALL".equals(parameters)) {
1085             return "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE";
1086         } else if ("FAST".equals(parameters)) {
1087             return "FLAGS INTERNALDATE RFC822.SIZE";
1088         } else if ("FULL".equals(parameters)) {
1089             return "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY";
1090         } else {
1091             return parameters;
1092         }
1093     }
1094 
1095     protected String[] getRequestedHeaders(String partIndexString) {
1096         if (partIndexString == null) {
1097             return null;
1098         } else {
1099             int startIndex = partIndexString.indexOf('(');
1100             int endIndex = partIndexString.indexOf(')');
1101             if (startIndex >= 0 && endIndex >= 0) {
1102                 return partIndexString.substring(startIndex + 1, endIndex).split(" ");
1103             } else {
1104                 return null;
1105             }
1106         }
1107     }
1108 
1109     protected void handleStore(String commandId, AbstractRangeIterator rangeIterator, String action, String flags) throws IOException {
1110         while (rangeIterator.hasNext()) {
1111             DavGatewayTray.switchIcon();
1112             ExchangeSession.Message message = rangeIterator.next();
1113             updateFlags(message, action, flags);
1114             sendClient("* " + (rangeIterator.getCurrentIndex()) + " FETCH (UID " + message.getImapUid() + " FLAGS (" + (message.getImapFlags()) + "))");
1115         }
1116         // auto expunge
1117         if (Settings.getBooleanProperty("davmail.imapAutoExpunge")) {
1118             if (expunge(false)) {
1119                 session.refreshFolder(currentFolder);
1120             }
1121         }
1122         sendClient(commandId + " OK STORE completed");
1123     }
1124 
1125     protected ExchangeSession.Condition buildConditions(SearchConditions conditions, ImapTokenizer tokens) throws IOException {
1126         ExchangeSession.MultiCondition condition = null;
1127         while (tokens.hasMoreTokens()) {
1128             String token = tokens.nextQuotedToken().toUpperCase();
1129             if (token.startsWith("(") && token.endsWith(")")) {
1130                 // quoted search param
1131                 if (condition == null) {
1132                     condition = session.and();
1133                 }
1134                 condition.add(buildConditions(conditions, new ImapTokenizer(token.substring(1, token.length() - 1))));
1135             } else if ("OR".equals(token)) {
1136                 condition = session.or();
1137             } else if (token.startsWith("OR ")) {
1138                 condition = appendOrSearchParams(token, conditions);
1139             } else if ("CHARSET".equals(token)) {
1140                 String charset = tokens.nextToken().toUpperCase();
1141                 if (!("ASCII".equals(charset) || "UTF-8".equals(charset) || "US-ASCII".equals(charset))) {
1142                     throw new IOException("Unsupported charset " + charset);
1143                 }
1144             } else {
1145                 if (condition == null) {
1146                     condition = session.and();
1147                 }
1148                 condition.add(appendSearchParam(tokens, token, conditions));
1149             }
1150         }
1151         return condition;
1152     }
1153 
1154 
1155     protected List<Long> handleSearch(ImapTokenizer tokens) throws IOException {
1156         List<Long> uidList = new ArrayList<>();
1157         List<Long> localMessagesUidList = null;
1158         SearchConditions conditions = new SearchConditions();
1159         ExchangeSession.Condition condition = buildConditions(conditions, tokens);
1160         session.refreshFolder(currentFolder);
1161         ExchangeSession.MessageList localMessages = currentFolder.searchMessages(condition);
1162         Iterator<ExchangeSession.Message> iterator;
1163         if (conditions.uidRange != null) {
1164             iterator = new UIDRangeIterator(localMessages, conditions.uidRange);
1165         } else if (conditions.indexRange != null) {
1166             // range iterator is on folder messages, not messages returned from search
1167             iterator = new RangeIterator(currentFolder.messages, conditions.indexRange);
1168             localMessagesUidList = new ArrayList<>();
1169             // build search result uid list
1170             for (ExchangeSession.Message message : localMessages) {
1171                 localMessagesUidList.add(message.getImapUid());
1172             }
1173         } else {
1174             iterator = localMessages.iterator();
1175         }
1176         while (iterator.hasNext()) {
1177             ExchangeSession.Message message = iterator.next();
1178             if ((conditions.flagged == null || message.flagged == conditions.flagged)
1179                     && (conditions.answered == null || message.answered == conditions.answered)
1180                     && (conditions.draft == null || message.draft == conditions.draft)
1181                     // range iterator: include messages available in search result
1182                     && (localMessagesUidList == null || localMessagesUidList.contains(message.getImapUid()))
1183                     && isNotExcluded(conditions.notUidRange, message.getImapUid())) {
1184                 uidList.add(message.getImapUid());
1185             }
1186         }
1187         return uidList;
1188     }
1189 
1190     /**
1191      * Check NOT UID condition.
1192      *
1193      * @param notUidRange excluded uid range
1194      * @param imapUid     current message imap uid
1195      * @return true if not excluded
1196      */
1197     private boolean isNotExcluded(String notUidRange, long imapUid) {
1198         if (notUidRange == null) {
1199             return true;
1200         }
1201         String imapUidAsString = String.valueOf(imapUid);
1202         for (String rangeValue : notUidRange.split(",")) {
1203             if (imapUidAsString.equals(rangeValue)) {
1204                 return false;
1205             }
1206         }
1207         return true;
1208     }
1209 
1210     protected void appendEnvelope(StringBuilder buffer, MessageWrapper message) throws IOException {
1211 
1212         try {
1213             MimeMessage mimeMessage = message.getMimeMessage();
1214             buffer.append(" ENVELOPE ");
1215             appendEnvelope(buffer, mimeMessage);
1216         } catch (MessagingException me) {
1217             DavGatewayTray.warn(me);
1218             // send fake envelope
1219             buffer.append(" ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)");
1220         }
1221     }
1222 
1223     private void appendEnvelope(StringBuilder buffer, MimePart mimePart) throws UnsupportedEncodingException, MessagingException {
1224         buffer.append('(');
1225         // Envelope for date, subject, from, sender, reply-to, to, cc, bcc,in-reply-to, message-id
1226         appendEnvelopeHeader(buffer, mimePart.getHeader("Date"));
1227         appendEnvelopeHeader(buffer, mimePart.getHeader("Subject"));
1228         appendMailEnvelopeHeader(buffer, mimePart.getHeader("From"));
1229         appendMailEnvelopeHeader(buffer, mimePart.getHeader("Sender"));
1230         appendMailEnvelopeHeader(buffer, mimePart.getHeader("Reply-To"));
1231         appendMailEnvelopeHeader(buffer, mimePart.getHeader("To"));
1232         appendMailEnvelopeHeader(buffer, mimePart.getHeader("CC"));
1233         appendMailEnvelopeHeader(buffer, mimePart.getHeader("BCC"));
1234         appendEnvelopeHeader(buffer, mimePart.getHeader("In-Reply-To"));
1235         appendEnvelopeHeader(buffer, mimePart.getHeader("Message-Id"));
1236         buffer.append(')');
1237     }
1238 
1239     protected void appendEnvelopeHeader(StringBuilder buffer, String[] value) throws UnsupportedEncodingException {
1240         if (buffer.charAt(buffer.length() - 1) != '(') {
1241             buffer.append(' ');
1242         }
1243         if (value != null && value.length > 0) {
1244             appendEnvelopeHeaderValue(buffer, MimeUtility.unfold(value[0]));
1245         } else {
1246             buffer.append("NIL");
1247         }
1248     }
1249 
1250     protected void appendMailEnvelopeHeader(StringBuilder buffer, String[] value) {
1251         buffer.append(' ');
1252         if (value != null && value.length > 0) {
1253             try {
1254                 String unfoldedValue = MimeUtility.unfold(value[0]);
1255                 InternetAddress[] addresses = InternetAddress.parseHeader(unfoldedValue, false);
1256                 if (addresses.length > 0) {
1257                     buffer.append('(');
1258                     for (InternetAddress address : addresses) {
1259                         buffer.append('(');
1260                         String personal = address.getPersonal();
1261                         if (personal != null) {
1262                             appendEnvelopeHeaderValue(buffer, personal);
1263                         } else {
1264                             buffer.append("NIL");
1265                         }
1266                         buffer.append(" NIL ");
1267                         String mail = address.getAddress();
1268                         int atIndex = mail.indexOf('@');
1269                         if (atIndex >= 0) {
1270                             buffer.append('"').append(mail, 0, atIndex).append('"');
1271                             buffer.append(' ');
1272                             buffer.append('"').append(mail.substring(atIndex + 1)).append('"');
1273                         } else {
1274                             buffer.append("NIL NIL");
1275                         }
1276                         buffer.append(')');
1277                     }
1278                     buffer.append(')');
1279                 } else {
1280                     buffer.append("NIL");
1281                 }
1282             } catch (AddressException | UnsupportedEncodingException e) {
1283                 DavGatewayTray.warn(e);
1284                 buffer.append("NIL");
1285             }
1286         } else {
1287             buffer.append("NIL");
1288         }
1289     }
1290 
1291     protected void appendEnvelopeHeaderValue(StringBuilder buffer, String value) throws UnsupportedEncodingException {
1292         if (value.indexOf('"') >= 0 || value.indexOf('\\') >= 0) {
1293             buffer.append('{');
1294             buffer.append(value.length());
1295             buffer.append("}\r\n");
1296             buffer.append(value);
1297         } else {
1298             buffer.append('"');
1299             buffer.append(MimeUtility.encodeText(value, "UTF-8", null));
1300             buffer.append('"');
1301         }
1302 
1303     }
1304 
1305     protected void appendBodyStructure(StringBuilder buffer, MessageWrapper message) throws IOException {
1306 
1307         buffer.append(" BODYSTRUCTURE ");
1308         try {
1309             MimeMessage mimeMessage = message.getMimeMessage();
1310             Object mimeBody = mimeMessage.getContent();
1311             if (mimeBody instanceof MimeMultipart) {
1312                 appendBodyStructure(buffer, (MimeMultipart) mimeBody);
1313             } else {
1314                 // no multipart, single body
1315                 appendBodyStructure(buffer, mimeMessage);
1316             }
1317         } catch (UnsupportedEncodingException | MessagingException e) {
1318             DavGatewayTray.warn(e);
1319             // failover: send default bodystructure
1320             buffer.append("(\"TEXT\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 0 0)");
1321         }
1322     }
1323 
1324     protected void appendBodyStructure(StringBuilder buffer, MimeMultipart multiPart) throws IOException, MessagingException {
1325         buffer.append('(');
1326 
1327         for (int i = 0; i < multiPart.getCount(); i++) {
1328             MimeBodyPart bodyPart = (MimeBodyPart) multiPart.getBodyPart(i);
1329             try {
1330                 Object mimeBody = bodyPart.getContent();
1331                 if (mimeBody instanceof MimeMultipart) {
1332                     appendBodyStructure(buffer, (MimeMultipart) mimeBody);
1333                 } else {
1334                     // no multipart, single body
1335                     appendBodyStructure(buffer, bodyPart);
1336                 }
1337             } catch (UnsupportedEncodingException e) {
1338                 LOGGER.warn(e);
1339                 // failover: send default bodystructure
1340                 buffer.append("(\"TEXT\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 0 0)");
1341             } catch (MessagingException me) {
1342                 DavGatewayTray.warn(me);
1343                 // failover: send default bodystructure
1344                 buffer.append("(\"TEXT\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 0 0)");
1345             }
1346         }
1347         int slashIndex = multiPart.getContentType().indexOf('/');
1348         if (slashIndex < 0) {
1349             throw new DavMailException("EXCEPTION_INVALID_CONTENT_TYPE", multiPart.getContentType());
1350         }
1351         int semiColonIndex = multiPart.getContentType().indexOf(';');
1352         if (semiColonIndex < 0) {
1353             buffer.append(" \"").append(multiPart.getContentType().substring(slashIndex + 1).toUpperCase()).append("\")");
1354         } else {
1355             buffer.append(" \"").append(multiPart.getContentType().substring(slashIndex + 1, semiColonIndex).trim().toUpperCase()).append("\")");
1356         }
1357     }
1358 
1359     protected void appendBodyStructure(StringBuilder buffer, MimePart bodyPart) throws IOException, MessagingException {
1360         String contentType = MimeUtility.unfold(bodyPart.getContentType());
1361         int slashIndex = contentType.indexOf('/');
1362         if (slashIndex < 0) {
1363             throw new DavMailException("EXCEPTION_INVALID_CONTENT_TYPE", contentType);
1364         }
1365         String type = contentType.substring(0, slashIndex).toUpperCase();
1366         buffer.append("(\"").append(type).append("\" \"");
1367         int semiColonIndex = contentType.indexOf(';');
1368         if (semiColonIndex < 0) {
1369             buffer.append(contentType.substring(slashIndex + 1).toUpperCase()).append("\" NIL");
1370         } else {
1371             // extended content type
1372             buffer.append(contentType.substring(slashIndex + 1, semiColonIndex).trim().toUpperCase()).append('\"');
1373             int charsetindex = contentType.indexOf("charset=");
1374             int nameindex = contentType.indexOf("name=");
1375             if (charsetindex >= 0 || nameindex >= 0) {
1376                 buffer.append(" (");
1377 
1378                 if (charsetindex >= 0) {
1379                     buffer.append("\"CHARSET\" ");
1380                     int charsetSemiColonIndex = contentType.indexOf(';', charsetindex);
1381                     int charsetEndIndex;
1382                     if (charsetSemiColonIndex > 0) {
1383                         charsetEndIndex = charsetSemiColonIndex;
1384                     } else {
1385                         charsetEndIndex = contentType.length();
1386                     }
1387                     String charSet = contentType.substring(charsetindex + "charset=".length(), charsetEndIndex);
1388                     if (!charSet.startsWith("\"")) {
1389                         buffer.append('"');
1390                     }
1391                     buffer.append(charSet.trim().toUpperCase());
1392                     if (!charSet.endsWith("\"")) {
1393                         buffer.append('"');
1394                     }
1395                 }
1396 
1397                 if (nameindex >= 0) {
1398                     if (charsetindex >= 0) {
1399                         buffer.append(' ');
1400                     }
1401 
1402                     buffer.append("\"NAME\" ");
1403                     int nameSemiColonIndex = contentType.indexOf(';', nameindex);
1404                     int nameEndIndex;
1405                     if (nameSemiColonIndex > 0) {
1406                         nameEndIndex = nameSemiColonIndex;
1407                     } else {
1408                         nameEndIndex = contentType.length();
1409                     }
1410                     String name = contentType.substring(nameindex + "name=".length(), nameEndIndex).trim();
1411                     if (!name.startsWith("\"")) {
1412                         buffer.append('"');
1413                     }
1414                     buffer.append(name.trim());
1415                     if (!name.endsWith("\"")) {
1416                         buffer.append('"');
1417                     }
1418                 }
1419                 buffer.append(')');
1420             } else {
1421                 buffer.append(" NIL");
1422             }
1423         }
1424         int bodySize = getBodyPartSize(bodyPart);
1425         appendBodyStructureValue(buffer, bodyPart.getContentID());
1426         appendBodyStructureValue(buffer, bodyPart.getDescription());
1427         appendBodyStructureValue(buffer, bodyPart.getEncoding());
1428         appendBodyStructureValue(buffer, bodySize);
1429 
1430         // line count not implemented in JavaMail, return fake line count
1431         int lineCount = bodySize / 80;
1432         if ("TEXT".equals(type)) {
1433             appendBodyStructureValue(buffer, lineCount);
1434         } else if ("MESSAGE".equals(type)) {
1435             Object bodyPartContent = bodyPart.getContent();
1436             if (bodyPartContent instanceof MimeMessage) {
1437                 MimeMessage innerMessage = (MimeMessage) bodyPartContent;
1438                 appendEnvelope(buffer, innerMessage);
1439                 appendBodyStructure(buffer, innerMessage);
1440                 appendBodyStructureValue(buffer, lineCount);
1441             } else {
1442                 // failover malformed message
1443                 appendBodyStructureValue(buffer, lineCount);
1444             }
1445         }
1446         buffer.append(')');
1447     }
1448 
1449     /**
1450      * Compute body part size with failover.
1451      * @param bodyPart MIME body part
1452      * @return body part size or 0 on error
1453      */
1454     private int getBodyPartSize(MimePart bodyPart) {
1455         int bodySize = 0;
1456         try {
1457             bodySize = bodyPart.getSize();
1458             if (bodySize == -1) {
1459                 // failover, try to get size
1460                 ByteArrayOutputStream baos = new ByteArrayOutputStream();
1461                 bodyPart.writeTo(baos);
1462                 bodySize = baos.size();
1463             }
1464         } catch (IOException | MessagingException e) {
1465             LOGGER.warn("Unable to get body part size " + e.getMessage(), e);
1466         }
1467         return bodySize;
1468     }
1469 
1470     protected void appendBodyStructureValue(StringBuilder buffer, String value) {
1471         if (value == null) {
1472             buffer.append(" NIL");
1473         } else {
1474             buffer.append(" \"").append(value.toUpperCase()).append('\"');
1475         }
1476     }
1477 
1478     protected void appendBodyStructureValue(StringBuilder buffer, int value) {
1479         if (value < 0) {
1480             // use 0 if we don't have a valid number
1481             buffer.append(" 0");
1482         } else {
1483             buffer.append(' ').append(value);
1484         }
1485     }
1486 
1487     protected void sendSubFolders(String command, String folderPath, boolean recursive, boolean wildcard, boolean specialOnly) throws IOException {
1488         try {
1489             List<ExchangeSession.Folder> folders = session.getSubFolders(folderPath, recursive, wildcard);
1490             for (ExchangeSession.Folder folder : folders) {
1491                 if (!specialOnly || folder.isSpecial()) {
1492                     sendClient("* " + command + " (" + folder.getFlags() + ") \"/\" \"" + encodeFolderPath(folder.folderPath) + '\"');
1493                 }
1494             }
1495         } catch (HttpForbiddenException e) {
1496             // access forbidden, ignore
1497             DavGatewayTray.debug(new BundleMessage("LOG_SUBFOLDER_ACCESS_FORBIDDEN", folderPath));
1498         } catch (HttpNotFoundException e) {
1499             // not found, ignore
1500             DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_NOT_FOUND", folderPath));
1501         } catch (HttpResponseException e) {
1502             // other errors, ignore
1503             DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_ACCESS_ERROR", folderPath, e.getMessage()));
1504         }
1505     }
1506 
1507     /**
1508      * client side search conditions
1509      */
1510     static final protected class SearchConditions {
1511         Boolean flagged;
1512         Boolean answered;
1513         Boolean draft;
1514         String indexRange;
1515         String uidRange;
1516         String notUidRange;
1517     }
1518 
1519     protected ExchangeSession.MultiCondition appendOrSearchParams(String token, SearchConditions conditions) throws IOException {
1520         ExchangeSession.MultiCondition orCondition = session.or();
1521         ImapTokenizer innerTokens = new ImapTokenizer(token);
1522         innerTokens.nextToken();
1523         while (innerTokens.hasMoreTokens()) {
1524             String innerToken = innerTokens.nextToken();
1525             orCondition.add(appendSearchParam(innerTokens, innerToken, conditions));
1526         }
1527         return orCondition;
1528     }
1529 
1530     protected ExchangeSession.Condition appendNotSearchParams(String token, SearchConditions conditions) throws IOException {
1531         ImapTokenizer innerTokens = new ImapTokenizer(token);
1532         ExchangeSession.Condition cond = buildConditions(conditions, innerTokens);
1533         if (cond == null || cond.isEmpty()) {
1534             return null;
1535         }
1536         return session.not(cond);
1537     }
1538 
1539     protected ExchangeSession.Condition appendSearchParam(ImapTokenizer tokens, String token, SearchConditions conditions) throws IOException {
1540         if ("NOT".equals(token)) {
1541             String nextToken = tokens.nextToken();
1542             if ("DELETED".equals(nextToken)) {
1543                 // conditions.deleted = Boolean.FALSE;
1544                 return session.isNull("deleted");
1545             } else if ("KEYWORD".equals(nextToken)) {
1546                 return appendNotSearchParams(nextToken + " " + tokens.nextToken(), conditions);
1547             } else if ("UID".equals(nextToken)) {
1548                 conditions.notUidRange = tokens.nextToken();
1549             } else {
1550                 return appendNotSearchParams(nextToken, conditions);
1551             }
1552         } else if (token.startsWith("OR ")) {
1553             return appendOrSearchParams(token, conditions);
1554         } else if ("SUBJECT".equals(token)) {
1555             return session.contains("subject", tokens.nextToken());
1556         } else if ("BODY".equals(token)) {
1557             return session.contains("body", tokens.nextToken());
1558         } else if ("TEXT".equals(token)) {
1559             String value = tokens.nextToken();
1560             return session.or(session.contains("body", value),
1561                     session.contains("subject", value),
1562                     session.contains("from", value),
1563                     session.contains("to", value),
1564                     session.contains("cc", value));
1565         } else if ("KEYWORD".equals(token)) {
1566             return session.isEqualTo("keywords", session.convertFlagToKeyword(tokens.nextToken()));
1567         } else if ("FROM".equals(token)) {
1568             return session.contains("from", tokens.nextToken());
1569         } else if ("TO".equals(token)) {
1570             return session.contains("to", tokens.nextToken());
1571         } else if ("CC".equals(token)) {
1572             return session.contains("cc", tokens.nextToken());
1573         } else if ("LARGER".equals(token)) {
1574             return session.gte("messageSize", tokens.nextToken());
1575         } else if ("SMALLER".equals(token)) {
1576             return session.lt("messageSize", tokens.nextToken());
1577         } else if (token.startsWith("SENT") || "SINCE".equals(token) || "BEFORE".equals(token) || "ON".equals(token)) {
1578             return appendDateSearchParam(tokens, token);
1579         } else if ("SEEN".equals(token)) {
1580             return session.isTrue("read");
1581         } else if ("UNSEEN".equals(token) || "NEW".equals(token)) {
1582             return session.isFalse("read");
1583         } else if ("DRAFT".equals(token)) {
1584             conditions.draft = Boolean.TRUE;
1585         } else if ("UNDRAFT".equals(token)) {
1586             conditions.draft = Boolean.FALSE;
1587         } else if ("DELETED".equals(token)) {
1588             // conditions.deleted = Boolean.TRUE;
1589             return session.isEqualTo("deleted", "1");
1590         } else if ("UNDELETED".equals(token) || "NOT DELETED".equals(token)) {
1591             // conditions.deleted = Boolean.FALSE;
1592             return session.isNull("deleted");
1593         } else if ("FLAGGED".equals(token)) {
1594             conditions.flagged = Boolean.TRUE;
1595         } else if ("UNFLAGGED".equals(token)) {
1596             conditions.flagged = Boolean.FALSE;
1597         } else if ("ANSWERED".equals(token)) {
1598             conditions.answered = Boolean.TRUE;
1599         } else if ("UNANSWERED".equals(token)) {
1600             conditions.answered = Boolean.FALSE;
1601         } else if ("HEADER".equals(token)) {
1602             String headerName = tokens.nextToken().toLowerCase();
1603             String value = tokens.nextToken();
1604             if ("message-id".equals(headerName) && !value.startsWith("<")) {
1605                 value = '<' + value + '>';
1606             }
1607             return session.headerIsEqualTo(headerName, value);
1608         } else if ("UID".equals(token)) {
1609             String range = tokens.nextToken();
1610             // ignore 1:* noop filter
1611             if (!"1:*".equals(range)) {
1612                 conditions.uidRange = range;
1613             }
1614         } else //noinspection StatementWithEmptyBody
1615             if ("OLD".equals(token) || "RECENT".equals(token) || "ALL".equals(token)) {
1616                 // ignore
1617             } else if (token.indexOf(':') >= 0 || token.matches("\\d+") || token.indexOf(',') >= 0) {
1618                 // range search
1619                 conditions.indexRange = token;
1620             } else {
1621                 throw new DavMailException("EXCEPTION_INVALID_SEARCH_PARAMETERS", token);
1622             }
1623         // client side search token
1624         return null;
1625     }
1626 
1627     protected ExchangeSession.Condition appendDateSearchParam(ImapTokenizer tokens, String token) throws IOException {
1628         Date startDate;
1629         Date endDate;
1630         SimpleDateFormat parser = new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH);
1631         parser.setTimeZone(ExchangeSession.GMT_TIMEZONE);
1632         String dateToken = tokens.nextToken();
1633         try {
1634             startDate = parser.parse(dateToken);
1635             Calendar calendar = Calendar.getInstance();
1636             calendar.setTime(startDate);
1637             calendar.add(Calendar.DAY_OF_MONTH, 1);
1638             endDate = calendar.getTime();
1639         } catch (ParseException e) {
1640             throw new DavMailException("EXCEPTION_INVALID_SEARCH_PARAMETERS", dateToken);
1641         }
1642         String searchAttribute;
1643         if (token.startsWith("SENT")) {
1644             searchAttribute = "date";
1645         } else {
1646             searchAttribute = "lastmodified";
1647         }
1648 
1649         if (token.endsWith("ON")) {
1650             return session.and(session.gt(searchAttribute, session.formatSearchDate(startDate)),
1651                     session.lt(searchAttribute, session.formatSearchDate(endDate)));
1652         } else if (token.endsWith("BEFORE")) {
1653             return session.lt(searchAttribute, session.formatSearchDate(startDate));
1654         } else if (token.endsWith("SINCE")) {
1655             return session.gte(searchAttribute, session.formatSearchDate(startDate));
1656         } else {
1657             throw new DavMailException("EXCEPTION_INVALID_SEARCH_PARAMETERS", dateToken);
1658         }
1659     }
1660 
1661     protected boolean expunge(boolean silent) throws IOException {
1662         boolean hasDeleted = false;
1663         if (currentFolder.messages != null) {
1664             int index = 1;
1665             for (ExchangeSession.Message message : currentFolder.messages) {
1666                 if (message.deleted) {
1667                     message.delete();
1668                     hasDeleted = true;
1669                     if (!silent) {
1670                         sendClient("* " + index + " EXPUNGE");
1671                     }
1672                 } else {
1673                     index++;
1674                 }
1675             }
1676         }
1677         return hasDeleted;
1678     }
1679 
1680     protected void updateFlags(ExchangeSession.Message message, String action, String flags) throws IOException {
1681         HashMap<String, String> properties = new HashMap<>();
1682         if ("-Flags".equalsIgnoreCase(action) || "-FLAGS.SILENT".equalsIgnoreCase(action)) {
1683             ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
1684             while (flagtokenizer.hasMoreTokens()) {
1685                 String flag = flagtokenizer.nextToken();
1686                 if ("\\Seen".equalsIgnoreCase(flag)) {
1687                     if (message.read) {
1688                         properties.put("read", "0");
1689                         message.read = false;
1690                     }
1691                 } else if ("\\Flagged".equalsIgnoreCase(flag)) {
1692                     if (message.flagged) {
1693                         properties.put("flagged", "0");
1694                         message.flagged = false;
1695                     }
1696                 } else if ("\\Deleted".equalsIgnoreCase(flag)) {
1697                     if (message.deleted) {
1698                         properties.put("deleted", null);
1699                         message.deleted = false;
1700                     }
1701                 } else if ("Junk".equalsIgnoreCase(flag)) {
1702                     if (message.junk) {
1703                         properties.put("junk", "0");
1704                         message.junk = false;
1705                     }
1706                 } else if ("$Forwarded".equalsIgnoreCase(flag)) {
1707                     if (message.forwarded) {
1708                         properties.put("forwarded", null);
1709                         message.forwarded = false;
1710                     }
1711                 } else if ("\\Answered".equalsIgnoreCase(flag)) {
1712                     if (message.answered) {
1713                         properties.put("answered", null);
1714                         message.answered = false;
1715                     }
1716                 } else //noinspection StatementWithEmptyBody
1717                     if ("\\Draft".equalsIgnoreCase(flag)) {
1718                         // ignore, draft is readonly after create
1719                     } else if (message.keywords != null) {
1720                         properties.put("keywords", message.removeFlag(flag));
1721                     }
1722             }
1723         } else if ("+Flags".equalsIgnoreCase(action) || "+FLAGS.SILENT".equalsIgnoreCase(action)) {
1724             ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
1725             while (flagtokenizer.hasMoreTokens()) {
1726                 String flag = flagtokenizer.nextToken();
1727                 if ("\\Seen".equalsIgnoreCase(flag)) {
1728                     if (!message.read) {
1729                         properties.put("read", "1");
1730                         message.read = true;
1731                     }
1732                 } else if ("\\Deleted".equalsIgnoreCase(flag)) {
1733                     if (!message.deleted) {
1734                         message.deleted = true;
1735                         properties.put("deleted", "1");
1736                     }
1737                 } else if ("\\Flagged".equalsIgnoreCase(flag)) {
1738                     if (!message.flagged) {
1739                         properties.put("flagged", "2");
1740                         message.flagged = true;
1741                     }
1742                 } else if ("\\Answered".equalsIgnoreCase(flag)) {
1743                     if (!message.answered) {
1744                         properties.put("answered", "102");
1745                         message.answered = true;
1746                     }
1747                 } else if ("$Forwarded".equalsIgnoreCase(flag)) {
1748                     if (!message.forwarded) {
1749                         properties.put("forwarded", "104");
1750                         message.forwarded = true;
1751                     }
1752                 } else if ("Junk".equalsIgnoreCase(flag)) {
1753                     if (!message.junk) {
1754                         properties.put("junk", "1");
1755                         message.junk = true;
1756                     }
1757                 } else //noinspection StatementWithEmptyBody
1758                     if ("\\Draft".equalsIgnoreCase(flag)) {
1759                         // ignore, draft is readonly after create
1760                     } else {
1761                         properties.put("keywords", message.addFlag(flag));
1762                     }
1763             }
1764         } else if ("FLAGS".equalsIgnoreCase(action) || "FLAGS.SILENT".equalsIgnoreCase(action)) {
1765             // flag list with default values
1766             boolean read = false;
1767             boolean deleted = false;
1768             boolean junk = false;
1769             boolean flagged = false;
1770             boolean answered = false;
1771             boolean forwarded = false;
1772             HashSet<String> keywords = null;
1773             // set flags from new flag list
1774             ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
1775             while (flagtokenizer.hasMoreTokens()) {
1776                 String flag = flagtokenizer.nextToken();
1777                 if ("\\Seen".equalsIgnoreCase(flag)) {
1778                     read = true;
1779                 } else if ("\\Deleted".equalsIgnoreCase(flag)) {
1780                     deleted = true;
1781                 } else if ("\\Flagged".equalsIgnoreCase(flag)) {
1782                     flagged = true;
1783                 } else if ("\\Answered".equalsIgnoreCase(flag)) {
1784                     answered = true;
1785                 } else if ("$Forwarded".equalsIgnoreCase(flag)) {
1786                     forwarded = true;
1787                 } else if ("Junk".equalsIgnoreCase(flag)) {
1788                     junk = true;
1789                 } else //noinspection StatementWithEmptyBody
1790                     if ("\\Draft".equalsIgnoreCase(flag)) {
1791                         // ignore, draft is readonly after create
1792                     } else {
1793                         if (keywords == null) {
1794                             keywords = new HashSet<>();
1795                         }
1796                         keywords.add(flag);
1797                     }
1798             }
1799             if (keywords != null) {
1800                 properties.put("keywords", message.setFlags(keywords));
1801             }
1802             if (read != message.read) {
1803                 message.read = read;
1804                 if (message.read) {
1805                     properties.put("read", "1");
1806                 } else {
1807                     properties.put("read", "0");
1808                 }
1809             }
1810             if (deleted != message.deleted) {
1811                 message.deleted = deleted;
1812                 if (message.deleted) {
1813                     properties.put("deleted", "1");
1814                 } else {
1815                     properties.put("deleted", null);
1816                 }
1817             }
1818             if (flagged != message.flagged) {
1819                 message.flagged = flagged;
1820                 if (message.flagged) {
1821                     properties.put("flagged", "2");
1822                 } else {
1823                     properties.put("flagged", "0");
1824                 }
1825             }
1826             if (answered != message.answered) {
1827                 message.answered = answered;
1828                 if (message.answered) {
1829                     properties.put("answered", "102");
1830                 } else if (!forwarded) {
1831                     // remove property only if not forwarded
1832                     properties.put("answered", null);
1833                 }
1834             }
1835             if (forwarded != message.forwarded) {
1836                 message.forwarded = forwarded;
1837                 if (message.forwarded) {
1838                     properties.put("forwarded", "104");
1839                 } else if (!answered) {
1840                     // remove property only if not answered
1841                     properties.put("forwarded", null);
1842                 }
1843             }
1844             if (junk != message.junk) {
1845                 message.junk = junk;
1846                 if (message.junk) {
1847                     properties.put("junk", "1");
1848                 } else {
1849                     properties.put("junk", "0");
1850                 }
1851             }
1852         }
1853         if (!properties.isEmpty()) {
1854             session.updateMessage(message, properties);
1855             // message is no longer recent
1856             message.recent = false;
1857         }
1858     }
1859 
1860     /**
1861      * Decode IMAP credentials
1862      *
1863      * @param tokens tokens
1864      * @throws IOException on error
1865      */
1866     protected void parseCredentials(ImapTokenizer tokens) throws IOException {
1867         if (tokens.hasMoreTokens()) {
1868             userName = tokens.nextToken();
1869         } else {
1870             throw new DavMailException("EXCEPTION_INVALID_CREDENTIALS");
1871         }
1872 
1873         if (tokens.hasMoreTokens()) {
1874             password = tokens.nextToken();
1875         } else {
1876             throw new DavMailException("EXCEPTION_INVALID_CREDENTIALS");
1877         }
1878     }
1879 
1880     /**
1881      * Filter to output only headers, also count full size
1882      */
1883     private static final class PartOutputStream extends FilterOutputStream {
1884         protected enum State {
1885             START, CR, CRLF, CRLFCR, BODY
1886         }
1887 
1888         private State state = State.START;
1889         private int size;
1890         private int bufferSize;
1891         private final boolean writeHeaders;
1892         private final boolean writeBody;
1893         private final int startIndex;
1894         private final int maxSize;
1895 
1896         private PartOutputStream(OutputStream os, boolean writeHeaders, boolean writeBody,
1897                                  int startIndex, int maxSize) {
1898             super(os);
1899             this.writeHeaders = writeHeaders;
1900             this.writeBody = writeBody;
1901             this.startIndex = startIndex;
1902             this.maxSize = maxSize;
1903         }
1904 
1905         @Override
1906         public void write(int b) throws IOException {
1907             size++;
1908             if (((state != State.BODY && writeHeaders) || (state == State.BODY && writeBody)) &&
1909                     (size > startIndex) && (bufferSize < maxSize)
1910             ) {
1911                 super.write(b);
1912                 bufferSize++;
1913             }
1914             if (state == State.START) {
1915                 if (b == '\r') {
1916                     state = State.CR;
1917                 }
1918             } else if (state == State.CR) {
1919                 if (b == '\n') {
1920                     state = State.CRLF;
1921                 } else {
1922                     state = State.START;
1923                 }
1924             } else if (state == State.CRLF) {
1925                 if (b == '\r') {
1926                     state = State.CRLFCR;
1927                 } else {
1928                     state = State.START;
1929                 }
1930             } else if (state == State.CRLFCR) {
1931                 if (b == '\n') {
1932                     state = State.BODY;
1933                 } else {
1934                     state = State.START;
1935                 }
1936             }
1937         }
1938     }
1939 
1940     /**
1941      * Partial output stream, start at startIndex and write maxSize bytes.
1942      */
1943     private static final class PartialOutputStream extends FilterOutputStream {
1944         private int size;
1945         private int bufferSize;
1946         private final int startIndex;
1947         private final int maxSize;
1948 
1949         private PartialOutputStream(OutputStream os, int startIndex, int maxSize) {
1950             super(os);
1951             this.startIndex = startIndex;
1952             this.maxSize = maxSize;
1953         }
1954 
1955         @Override
1956         public void write(int b) throws IOException {
1957             size++;
1958             if ((size > startIndex) && (bufferSize < maxSize)) {
1959                 super.write(b);
1960                 bufferSize++;
1961             }
1962         }
1963     }
1964 
1965     protected abstract static class AbstractRangeIterator implements Iterator<ExchangeSession.Message> {
1966         ExchangeSession.MessageList messages;
1967         int currentIndex;
1968 
1969         protected int getCurrentIndex() {
1970             return currentIndex;
1971         }
1972     }
1973 
1974     protected static class UIDRangeIterator extends AbstractRangeIterator {
1975         final String[] ranges;
1976         int currentRangeIndex;
1977         long startUid;
1978         long endUid;
1979 
1980         protected UIDRangeIterator(ExchangeSession.MessageList messages, String value) {
1981             this.messages = messages;
1982             ranges = value.split(",");
1983         }
1984 
1985         protected long convertToLong(String value) {
1986             if ("*".equals(value)) {
1987                 return Long.MAX_VALUE;
1988             } else {
1989                 return Long.parseLong(value);
1990             }
1991         }
1992 
1993         protected void skipToNextRangeStartUid() {
1994             if (currentRangeIndex < ranges.length) {
1995                 String currentRange = ranges[currentRangeIndex++];
1996                 int colonIndex = currentRange.indexOf(':');
1997                 if (colonIndex > 0) {
1998                     startUid = convertToLong(currentRange.substring(0, colonIndex));
1999                     endUid = convertToLong(currentRange.substring(colonIndex + 1));
2000                     if (endUid < startUid) {
2001                         long swap = endUid;
2002                         endUid = startUid;
2003                         startUid = swap;
2004                     }
2005                 } else if ("*".equals(currentRange)) {
2006                     startUid = endUid = messages.get(messages.size() - 1).getImapUid();
2007                 } else {
2008                     startUid = endUid = convertToLong(currentRange);
2009                 }
2010                 while (currentIndex < messages.size() && messages.get(currentIndex).getImapUid() < startUid) {
2011                     currentIndex++;
2012                 }
2013             } else {
2014                 currentIndex = messages.size();
2015             }
2016         }
2017 
2018         protected boolean hasNextInRange() {
2019             return hasNextIndex() && messages.get(currentIndex).getImapUid() <= endUid;
2020         }
2021 
2022         protected boolean hasNextIndex() {
2023             return currentIndex < messages.size();
2024         }
2025 
2026         protected boolean hasNextRange() {
2027             return currentRangeIndex < ranges.length;
2028         }
2029 
2030         public boolean hasNext() {
2031             boolean hasNextInRange = hasNextInRange();
2032             // if has next range and current index after current range end, reset index
2033             if (hasNextRange() && !hasNextInRange) {
2034                 currentIndex = 0;
2035             }
2036             while (hasNextIndex() && !hasNextInRange) {
2037                 skipToNextRangeStartUid();
2038                 hasNextInRange = hasNextInRange();
2039             }
2040             return hasNextIndex();
2041         }
2042 
2043         public ExchangeSession.Message next() {
2044             ExchangeSession.Message message = messages.get(currentIndex++);
2045             long uid = message.getImapUid();
2046             if (uid < startUid || uid > endUid) {
2047                 throw new NoSuchElementException("Message uid " + uid + " not in range " + startUid + ':' + endUid);
2048             }
2049             return message;
2050         }
2051 
2052         public void remove() {
2053             throw new UnsupportedOperationException();
2054         }
2055     }
2056 
2057     protected static class RangeIterator extends AbstractRangeIterator {
2058         final String[] ranges;
2059         int currentRangeIndex;
2060         long startUid;
2061         long endUid;
2062 
2063         protected RangeIterator(ExchangeSession.MessageList messages, String value) {
2064             this.messages = messages;
2065             ranges = value.split(",");
2066         }
2067 
2068         protected long convertToLong(String value) {
2069             if ("*".equals(value)) {
2070                 return Long.MAX_VALUE;
2071             } else {
2072                 return Long.parseLong(value);
2073             }
2074         }
2075 
2076         protected void skipToNextRangeStart() {
2077             if (currentRangeIndex < ranges.length) {
2078                 String currentRange = ranges[currentRangeIndex++];
2079                 int colonIndex = currentRange.indexOf(':');
2080                 if (colonIndex > 0) {
2081                     startUid = convertToLong(currentRange.substring(0, colonIndex));
2082                     endUid = convertToLong(currentRange.substring(colonIndex + 1));
2083                     if (endUid < startUid) {
2084                         long swap = endUid;
2085                         endUid = startUid;
2086                         startUid = swap;
2087                     }
2088                 } else if ("*".equals(currentRange)) {
2089                     startUid = endUid = messages.size();
2090                 } else {
2091                     startUid = endUid = convertToLong(currentRange);
2092                 }
2093                 while (currentIndex < messages.size() && (currentIndex + 1) < startUid) {
2094                     currentIndex++;
2095                 }
2096             } else {
2097                 currentIndex = messages.size();
2098             }
2099         }
2100 
2101         protected boolean hasNextInRange() {
2102             return hasNextIndex() && currentIndex < endUid;
2103         }
2104 
2105         protected boolean hasNextIndex() {
2106             return currentIndex < messages.size();
2107         }
2108 
2109         protected boolean hasNextRange() {
2110             return currentRangeIndex < ranges.length;
2111         }
2112 
2113         public boolean hasNext() {
2114             boolean hasNextInRange = hasNextInRange();
2115             // if has next range and current index after current range end, reset index
2116             if (hasNextRange() && !hasNextInRange) {
2117                 currentIndex = 0;
2118             }
2119             while (hasNextIndex() && !hasNextInRange) {
2120                 skipToNextRangeStart();
2121                 hasNextInRange = hasNextInRange();
2122             }
2123             return hasNextIndex();
2124         }
2125 
2126         public ExchangeSession.Message next() {
2127             if (currentIndex >= messages.size()) {
2128                 throw new NoSuchElementException();
2129             }
2130             return messages.get(currentIndex++);
2131         }
2132 
2133         public void remove() {
2134             throw new UnsupportedOperationException();
2135         }
2136     }
2137 
2138     static protected class ImapTokenizer {
2139         char[] value;
2140         int startIndex;
2141         Stack<Character> quotes = new Stack<>();
2142 
2143         ImapTokenizer(String value) {
2144             this.value = value.toCharArray();
2145         }
2146 
2147         public String nextToken() {
2148             // Get next token without removing quotes ", {} or ()
2149             String token = nextQuotedToken();
2150             // note: literal strings not handled here.
2151             if( !token.isEmpty() && '"' == token.charAt(0) ) {
2152                 // token is quoted string.
2153                 try {
2154                     token = StringUtil.parseQuotedImapString(token);
2155                 } catch (ParseException e) {
2156                     LOGGER.warn("Invalid quoted token: "+token);
2157                     token = StringUtil.removeQuotes(token);
2158                 }
2159             } else {
2160                 // use the general method previously also used;
2161                 // for example unquotes a list. I guess naming could be made better in the future.
2162                 token = StringUtil.removeQuotes(token);
2163             }
2164             return token;
2165         }
2166 
2167         protected boolean isQuote(char character) {
2168             return character == '"' || character == '(' || character == ')' ||
2169                     character == '[' || character == ']' || character == '\\';
2170         }
2171 
2172         public boolean hasMoreTokens() {
2173             return startIndex < value.length;
2174         }
2175 
2176         public String nextQuotedToken() {
2177             int currentIndex = startIndex;
2178             while (currentIndex < value.length) {
2179                 char currentChar = value[currentIndex];
2180                 if (currentChar == ' ' && quotes.isEmpty()) {
2181                     break;
2182                 } else if (!quotes.isEmpty() && quotes.peek() == '\\') {
2183                     // just skip
2184                     quotes.pop();
2185                 } else if (isQuote(currentChar)) {
2186                     if (quotes.isEmpty()) {
2187                         quotes.push(currentChar);
2188                     } else {
2189                         char currentQuote = quotes.peek();
2190                         if (currentChar == '\\') {
2191                             quotes.push(currentChar);
2192                         } else if (currentQuote == '"' && currentChar == '"' ||
2193                                 currentQuote == '(' && currentChar == ')' ||
2194                                 currentQuote == '[' && currentChar == ']'
2195                         ) {
2196                             // end quote
2197                             quotes.pop();
2198                         } else {
2199                             quotes.push(currentChar);
2200                         }
2201                     }
2202                 }
2203                 currentIndex++;
2204             }
2205             String result = new String(value, startIndex, currentIndex - startIndex);
2206             startIndex = currentIndex + 1;
2207             return result;
2208         }
2209     }
2210 
2211 }