View Javadoc
1   /*
2    * DavMail POP/IMAP/SMTP/CalDav/LDAP Exchange Gateway
3    * Copyright (C) 2009  Mickael Guessant
4    *
5    * This program is free software; you can redistribute it and/or
6    * modify it under the terms of the GNU General Public License
7    * as published by the Free Software Foundation; either version 2
8    * of the License, or (at your option) any later version.
9    *
10   * This program is distributed in the hope that it will be useful,
11   * but WITHOUT ANY WARRANTY; without even the implied warranty of
12   * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
13   * GNU General Public License for more details.
14   *
15   * You should have received a copy of the GNU General Public License
16   * along with this program; if not, write to the Free Software
17   * Foundation, Inc., 51 Franklin Street, Fifth Floor, Boston, MA  02110-1301, USA.
18   */
19  package davmail.imap;
20  
21  import com.sun.mail.imap.protocol.BASE64MailboxDecoder;
22  import com.sun.mail.imap.protocol.BASE64MailboxEncoder;
23  import davmail.AbstractConnection;
24  import davmail.BundleMessage;
25  import davmail.DavGateway;
26  import davmail.Settings;
27  import davmail.exception.DavMailException;
28  import davmail.exception.HttpForbiddenException;
29  import davmail.exchange.NetworkDownException;
30  import davmail.exception.HttpNotFoundException;
31  import davmail.exception.InsufficientStorageException;
32  import davmail.exchange.ExchangeSession;
33  import davmail.exchange.ExchangeSessionFactory;
34  import davmail.exchange.MessageCreateThread;
35  import davmail.exchange.MessageLoadThread;
36  import davmail.ui.tray.DavGatewayTray;
37  import davmail.util.IOUtil;
38  import davmail.util.StringUtil;
39  import org.apache.http.client.HttpResponseException;
40  import org.apache.log4j.Logger;
41  
42  import javax.mail.MessagingException;
43  import javax.mail.internet.AddressException;
44  import javax.mail.internet.InternetAddress;
45  import javax.mail.internet.MimeBodyPart;
46  import javax.mail.internet.MimeMessage;
47  import javax.mail.internet.MimeMultipart;
48  import javax.mail.internet.MimePart;
49  import javax.mail.internet.MimeUtility;
50  import javax.mail.util.SharedByteArrayInputStream;
51  import javax.net.ssl.SSLException;
52  import java.io.ByteArrayOutputStream;
53  import java.io.FilterOutputStream;
54  import java.io.IOException;
55  import java.io.InputStream;
56  import java.io.OutputStream;
57  import java.io.UnsupportedEncodingException;
58  import java.net.Socket;
59  import java.net.SocketException;
60  import java.net.SocketTimeoutException;
61  import java.net.UnknownHostException;
62  import java.nio.charset.StandardCharsets;
63  import java.text.ParseException;
64  import java.text.SimpleDateFormat;
65  import java.util.ArrayList;
66  import java.util.Calendar;
67  import java.util.Date;
68  import java.util.Enumeration;
69  import java.util.HashMap;
70  import java.util.HashSet;
71  import java.util.Set;
72  import java.util.Iterator;
73  import java.util.List;
74  import java.util.Locale;
75  import java.util.NoSuchElementException;
76  import java.util.ArrayDeque;
77  import java.util.TreeMap;
78  import java.util.UUID;
79  import java.util.concurrent.Executors;
80  import java.util.concurrent.ScheduledExecutorService;
81  import java.util.concurrent.ScheduledFuture;
82  import java.util.concurrent.TimeUnit;
83  
84  /**
85   * Dav Gateway IMAP connection implementation.
86   */
87  public class ImapConnection extends AbstractConnection {
88      private static final Logger LOGGER = Logger.getLogger(ImapConnection.class);
89  
90      protected String baseMailboxPath;
91      ExchangeSession.Folder currentFolder;
92  
93      /**
94       * Initialize the streams and start the thread.
95       *
96       * @param clientSocket IMAP client socket
97       */
98      public ImapConnection(Socket clientSocket) {
99          super(ImapConnection.class.getSimpleName(), clientSocket, "UTF-8");
100     }
101 
102     @Override
103     public void run() {
104         final String capabilities;
105         int imapIdleDelay = Settings.getIntProperty("davmail.imapIdleDelay") * 60;
106         if (imapIdleDelay > 0) {
107             capabilities = "CAPABILITY IMAP4REV1 AUTH=LOGIN IDLE MOVE SPECIAL-USE UIDPLUS";
108         } else {
109             capabilities = "CAPABILITY IMAP4REV1 AUTH=LOGIN MOVE SPECIAL-USE UIDPLUS";
110         }
111 
112         String line;
113         String commandId = null;
114         ImapTokenizer tokens;
115         try {
116             ExchangeSessionFactory.checkConfig();
117             sendClient("* OK [" + capabilities + "] IMAP4rev1 DavMail " + DavGateway.getCurrentVersion() + " server ready");
118             for (; ; ) {
119                 line = readClient();
120                 // unable to read line, connection closed ?
121                 if (line == null) {
122                     break;
123                 }
124 
125                 tokens = new ImapTokenizer(line);
126                 if (tokens.hasMoreTokens()) {
127                     commandId = tokens.nextToken();
128 
129                     checkInfiniteLoop(line);
130 
131                     if (tokens.hasMoreTokens()) {
132                         String command = tokens.nextToken();
133 
134                         if ("LOGOUT".equalsIgnoreCase(command)) {
135                             sendClient("* BYE Closing connection");
136                             sendClient(commandId + " OK LOGOUT completed");
137                             break;
138                         }
139                         if ("capability".equalsIgnoreCase(command)) {
140                             sendClient("* " + capabilities);
141                             sendClient(commandId + " OK CAPABILITY completed");
142                         } else if ("login".equalsIgnoreCase(command)) {
143                             parseCredentials(tokens);
144                             // detect shared mailbox access
145                             splitUserName();
146                             try {
147                                 session = ExchangeSessionFactory.getInstance(userName, password);
148                                 logConnection("LOGON", userName);
149                                 sendClient(commandId + " OK Authenticated");
150                                 state = State.AUTHENTICATED;
151                             } catch (NetworkDownException e) {
152                                 LOGGER.warn(e.getMessage());
153                                 sendClient("* BYE [NetworkDown] " + ((e.getMessage() == null) ? e.toString() : e.getMessage()).replaceAll("\\n", " "));
154                                 break;
155                             } catch (Exception e) {
156                                 logConnection("FAILED", userName);
157                                 DavGatewayTray.error(e);
158                                 if (Settings.getBooleanProperty("davmail.enableKerberos")) {
159                                     sendClient(commandId + " NO LOGIN Kerberos authentication failed");
160                                 } else {
161                                     sendClient(commandId + " NO LOGIN failed");
162                                 }
163                                 state = State.INITIAL;
164                             }
165                         } else if ("AUTHENTICATE".equalsIgnoreCase(command)) {
166                             if (tokens.hasMoreTokens()) {
167                                 String authenticationMethod = tokens.nextToken();
168                                 if ("LOGIN".equalsIgnoreCase(authenticationMethod)) {
169                                     try {
170                                         sendClient("+ " + IOUtil.encodeBase64AsString("Username:"));
171                                         state = State.LOGIN;
172                                         userName = IOUtil.decodeBase64AsString(readClient());
173                                         // detect shared mailbox access
174                                         splitUserName();
175                                         sendClient("+ " + IOUtil.encodeBase64AsString("Password:"));
176                                         state = State.PASSWORD;
177                                         password = IOUtil.decodeBase64AsString(readClient());
178                                         session = ExchangeSessionFactory.getInstance(userName, password);
179                                         logConnection("LOGON", userName);
180                                         sendClient(commandId + " OK Authenticated");
181                                         state = State.AUTHENTICATED;
182                                     } catch (NetworkDownException e) {
183                                         LOGGER.warn(e.getMessage());
184                                         sendClient("* BYE [NetworkDown] " + ((e.getMessage() == null) ? e.toString() : e.getMessage()).replaceAll("\\n", " "));
185                                         break;
186                                     } catch (Exception e) {
187                                         logConnection("FAILED", userName);
188                                         DavGatewayTray.error(e);
189                                         sendClient(commandId + " NO LOGIN failed");
190                                         state = State.INITIAL;
191                                     }
192                                 } else {
193                                     sendClient(commandId + " NO unsupported authentication method");
194                                 }
195                             } else {
196                                 sendClient(commandId + " BAD authentication method required");
197                             }
198                         } else {
199                             if (state != State.AUTHENTICATED) {
200                                 sendClient(commandId + " BAD command authentication required");
201                             } else {
202                                 // check for expired session
203                                 session = ExchangeSessionFactory.getInstance(session, userName, password);
204                                 if ("lsub".equalsIgnoreCase(command) || "list".equalsIgnoreCase(command)) {
205                                     if (tokens.hasMoreTokens()) {
206                                         String token = tokens.nextToken();
207                                         // handle list-extended selection option
208                                         // see https://www.rfc-editor.org/rfc/rfc6154 5.2
209                                         boolean specialOnly = token.contains("SPECIAL-USE");
210                                         if (specialOnly && tokens.hasMoreTokens()) {
211                                             token = tokens.nextToken();
212                                         }
213 
214                                         if (tokens.hasMoreTokens()) {
215                                             String folderQuery = buildFolderPath(token) + decodeFolderPath(tokens.nextToken());
216                                             if (folderQuery.endsWith("%/%") && !"/%/%".equals(folderQuery)) {
217                                                 List<ExchangeSession.Folder> folders = session.getSubFolders(folderQuery.substring(0, folderQuery.length() - 3), false, false);
218                                                 for (ExchangeSession.Folder folder : folders) {
219                                                     sendClient("* " + command + " (" + folder.getFlags() + ") \"/\" \"" + encodeFolderPath(folder.folderPath) + '\"');
220                                                     sendSubFolders(command, folder.folderPath, false, false, specialOnly);
221                                                 }
222                                                 sendClient(commandId + " OK " + command + " completed");
223                                             } else if (folderQuery.endsWith("%") || folderQuery.endsWith("*")) {
224                                                 if ("/*".equals(folderQuery) || "/%".equals(folderQuery) || "/%/%".equals(folderQuery)) {
225                                                     folderQuery = folderQuery.substring(1);
226                                                     if ("%/%".equals(folderQuery)) {
227                                                         folderQuery = folderQuery.substring(0, folderQuery.length() - 2);
228                                                     }
229                                                     sendClient("* " + command + " (\\HasChildren) \"/\" \"/public\"");
230                                                 }
231                                                 if ("*%".equals(folderQuery)) {
232                                                     folderQuery = "*";
233                                                 }
234                                                 boolean wildcard = folderQuery.endsWith("%") && !folderQuery.contains("/") && !folderQuery.equals("%");
235                                                 boolean recursive = folderQuery.endsWith("*");
236                                                 sendSubFolders(command, folderQuery.substring(0, folderQuery.length() - 1), recursive, wildcard,
237                                                         specialOnly);
238                                                 sendClient(commandId + " OK " + command + " completed");
239                                             } else {
240                                                 ExchangeSession.Folder folder = null;
241                                                 try {
242                                                     folder = session.getFolder(folderQuery);
243                                                 } catch (HttpForbiddenException e) {
244                                                     // access forbidden, ignore
245                                                     DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_ACCESS_FORBIDDEN", folderQuery));
246                                                 } catch (HttpNotFoundException e) {
247                                                     // not found, ignore
248                                                     DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_NOT_FOUND", folderQuery));
249                                                 } catch (HttpResponseException e) {
250                                                     // other errors, ignore
251                                                     DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_ACCESS_ERROR", folderQuery, e.getMessage()));
252                                                 }
253                                                 if (folder != null) {
254                                                     sendClient("* " + command + " (" + folder.getFlags() + ") \"/\" \"" + encodeFolderPath(folder.folderPath) + '\"');
255                                                     sendClient(commandId + " OK " + command + " completed");
256                                                 } else {
257                                                     sendClient(commandId + " NO Folder not found");
258                                                 }
259                                             }
260                                         } else {
261                                             sendClient(commandId + " BAD missing folder argument");
262                                         }
263                                     } else {
264                                         sendClient(commandId + " BAD missing folder argument");
265                                     }
266                                 } else if ("select".equalsIgnoreCase(command) || "examine".equalsIgnoreCase(command)) {
267                                     if (tokens.hasMoreTokens()) {
268                                         @SuppressWarnings({"NonConstantStringShouldBeStringBuffer"})
269                                         String folderName = buildFolderPath(tokens.nextToken());
270                                         try {
271                                             currentFolder = session.getFolder(folderName);
272                                             loadFolder(currentFolder);
273                                             sendClient("* " + currentFolder.count() + " EXISTS");
274                                             sendClient("* " + currentFolder.recent + " RECENT");
275                                             sendClient("* OK [UIDVALIDITY 1]");
276                                             if (currentFolder.count() == 0) {
277                                                 sendClient("* OK [UIDNEXT 1]");
278                                             } else {
279                                                 sendClient("* OK [UIDNEXT " + currentFolder.getUidNext() + ']');
280                                             }
281                                             sendClient("* FLAGS (\\Answered \\Deleted \\Draft \\Flagged \\Seen $Forwarded Junk)");
282                                             sendClient("* OK [PERMANENTFLAGS (\\Answered \\Deleted \\Draft \\Flagged \\Seen $Forwarded Junk \\*)]");
283                                             if ("select".equalsIgnoreCase(command)) {
284                                                 sendClient(commandId + " OK [READ-WRITE] " + command + " completed");
285                                             } else {
286                                                 sendClient(commandId + " OK [READ-ONLY] " + command + " completed");
287                                             }
288                                         } catch (HttpNotFoundException e) {
289                                             sendClient(commandId + " NO Not found");
290                                         } catch (HttpForbiddenException e) {
291                                             sendClient(commandId + " NO Forbidden");
292                                         }
293                                     } else {
294                                         sendClient(commandId + " BAD command unrecognized");
295                                     }
296                                 } else if ("expunge".equalsIgnoreCase(command)) {
297                                     if (expunge(false)) {
298                                         // need to refresh folder to avoid 404 errors
299                                         session.refreshFolder(currentFolder);
300                                     }
301                                     sendClient(commandId + " OK " + command + " completed");
302                                 } else if ("close".equalsIgnoreCase(command)) {
303                                     expunge(true);
304                                     // deselect folder
305                                     currentFolder = null;
306                                     sendClient(commandId + " OK " + command + " completed");
307                                 } else if ("create".equalsIgnoreCase(command)) {
308                                     if (tokens.hasMoreTokens()) {
309                                         session.createMessageFolder(buildFolderPath(tokens.nextToken()));
310                                         sendClient(commandId + " OK folder created");
311                                     } else {
312                                         sendClient(commandId + " BAD missing create argument");
313                                     }
314                                 } else if ("rename".equalsIgnoreCase(command)) {
315                                     String folderName = buildFolderPath(tokens.nextToken());
316                                     String targetName = buildFolderPath(tokens.nextToken());
317                                     try {
318                                         session.moveFolder(folderName, targetName);
319                                         sendClient(commandId + " OK rename completed");
320                                     } catch (HttpResponseException e) {
321                                         sendClient(commandId + " NO " + e.getMessage());
322                                     }
323                                 } else if ("delete".equalsIgnoreCase(command)) {
324                                     String folderName = buildFolderPath(tokens.nextToken());
325                                     try {
326                                         session.deleteFolder(folderName);
327                                         sendClient(commandId + " OK folder deleted");
328                                     } catch (HttpResponseException e) {
329                                         sendClient(commandId + " NO " + e.getMessage());
330                                     }
331                                 } else if ("uid".equalsIgnoreCase(command)) {
332                                     if (tokens.hasMoreTokens()) {
333                                         String subcommand = tokens.nextToken();
334                                         if ("fetch".equalsIgnoreCase(subcommand)) {
335                                             if (currentFolder == null) {
336                                                 sendClient(commandId + " NO no folder selected");
337                                             } else {
338                                                 String ranges = tokens.nextToken();
339                                                 if (ranges == null) {
340                                                     sendClient(commandId + " BAD missing range parameter");
341                                                 } else {
342                                                     String parameters = null;
343                                                     if (tokens.hasMoreTokens()) {
344                                                         parameters = tokens.nextToken();
345                                                     }
346                                                     UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, ranges);
347                                                     while (uidRangeIterator.hasNext()) {
348                                                         DavGatewayTray.switchIcon();
349                                                         ExchangeSession.Message message = uidRangeIterator.next();
350                                                         try {
351                                                             handleFetch(message, uidRangeIterator.currentIndex, parameters);
352                                                         } catch (HttpNotFoundException e) {
353                                                             LOGGER.warn("Ignore missing message " + uidRangeIterator.currentIndex);
354                                                         } catch (SocketException | SocketTimeoutException |
355                                                                  SSLException | UnknownHostException | NetworkDownException e) {
356                                                             // client closed connection or network error
357                                                             throw e;
358                                                         } catch (IOException e) {
359                                                             DavGatewayTray.log(e);
360                                                             LOGGER.warn("Ignore broken message " + uidRangeIterator.currentIndex + ' ' + e.getMessage());
361                                                         }
362                                                     }
363                                                     sendClient(commandId + " OK UID FETCH completed");
364                                                 }
365                                             }
366 
367                                         } else if ("search".equalsIgnoreCase(subcommand)) {
368                                             List<Long> uidList = handleSearch(tokens);
369                                             StringBuilder buffer = new StringBuilder("* SEARCH");
370                                             for (long uid : uidList) {
371                                                 buffer.append(' ');
372                                                 buffer.append(uid);
373                                             }
374                                             sendClient(buffer.toString());
375                                             sendClient(commandId + " OK SEARCH completed");
376 
377                                         } else if ("store".equalsIgnoreCase(subcommand)) {
378                                             UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, tokens.nextToken());
379                                             String action = tokens.nextToken();
380                                             String flags = tokens.nextToken();
381                                             handleStore(commandId, uidRangeIterator, action, flags);
382                                         } else if ("copy".equalsIgnoreCase(subcommand) || "move".equalsIgnoreCase(subcommand)) {
383                                             try {
384                                                 UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, tokens.nextToken());
385                                                 String targetName = buildFolderPath(tokens.nextToken());
386                                                 if (!uidRangeIterator.hasNext()) {
387                                                     sendClient(commandId + " NO " + "No message found");
388                                                 } else {
389                                                     ArrayList<ExchangeSession.Message> messages = new ArrayList<>();
390                                                     while (uidRangeIterator.hasNext()) {
391                                                         messages.add(uidRangeIterator.next());
392                                                     }
393                                                     if ("copy".equalsIgnoreCase(subcommand)) {
394                                                         session.copyMessages(messages, targetName);
395                                                     } else {
396                                                         session.moveMessages(messages, targetName);
397                                                         refreshCurrentFolder();
398                                                     }
399                                                     sendClient(commandId + " OK " + subcommand + " completed");
400                                                 }
401                                             } catch (HttpNotFoundException e) {
402                                                 sendClient(commandId + " NO [TRYCREATE] " + e.getMessage());
403                                             } catch (HttpResponseException e) {
404                                                 sendClient(commandId + " NO " + e.getMessage());
405                                             }
406                                         } else if ("expunge".equalsIgnoreCase(subcommand)) {
407                                             // RFC 4315 UID EXPUNGE: expunge only \Deleted messages within the UID set
408                                             if (currentFolder == null) {
409                                                 sendClient(commandId + " NO no folder selected");
410                                             } else if (!tokens.hasMoreTokens()) {
411                                                 sendClient(commandId + " BAD missing UID set parameter");
412                                             } else {
413                                                 UIDRangeIterator uidRangeIterator = new UIDRangeIterator(currentFolder.messages, tokens.nextToken());
414                                                 Set<Long> uidSet = new HashSet<>();
415                                                 while (uidRangeIterator.hasNext()) {
416                                                     uidSet.add(uidRangeIterator.next().getImapUid());
417                                                 }
418                                                 if (expunge(false, uidSet)) {
419                                                     session.refreshFolder(currentFolder);
420                                                 }
421                                                 sendClient(commandId + " OK UID EXPUNGE completed");
422                                             }
423                                         } else {
424                                             sendClient(commandId + " BAD unsupported UID subcommand " + subcommand);
425                                         }
426                                     } else {
427                                         sendClient(commandId + " BAD command unrecognized");
428                                     }
429                                 } else if ("search".equalsIgnoreCase(command)) {
430                                     if (currentFolder == null) {
431                                         sendClient(commandId + " NO no folder selected");
432                                     } else {
433                                         List<Long> uidList = handleSearch(tokens);
434                                         if (uidList.isEmpty()) {
435                                             sendClient("* SEARCH");
436                                         } else {
437                                             int currentIndex = 0;
438                                             StringBuilder buffer = new StringBuilder("* SEARCH");
439                                             for (ExchangeSession.Message message : currentFolder.messages) {
440                                                 currentIndex++;
441                                                 if (uidList.contains(message.getImapUid())) {
442                                                     buffer.append(' ');
443                                                     buffer.append(currentIndex);
444                                                 }
445                                             }
446                                             sendClient(buffer.toString());
447                                         }
448                                         sendClient(commandId + " OK SEARCH completed");
449                                     }
450                                 } else if ("fetch".equalsIgnoreCase(command)) {
451                                     if (currentFolder == null) {
452                                         sendClient(commandId + " NO no folder selected");
453                                     } else {
454                                         RangeIterator rangeIterator = new RangeIterator(currentFolder.messages, tokens.nextToken());
455                                         String parameters = null;
456                                         if (tokens.hasMoreTokens()) {
457                                             parameters = tokens.nextToken();
458                                         }
459                                         while (rangeIterator.hasNext()) {
460                                             DavGatewayTray.switchIcon();
461                                             ExchangeSession.Message message = rangeIterator.next();
462                                             try {
463                                                 handleFetch(message, rangeIterator.currentIndex, parameters);
464                                             } catch (HttpNotFoundException e) {
465                                                 LOGGER.warn("Ignore missing message " + rangeIterator.currentIndex);
466                                             } catch (SocketException | SocketTimeoutException | SSLException | UnknownHostException | NetworkDownException e) {
467                                                 // client closed connection or network error, rethrow exception
468                                                 throw e;
469                                             } catch (IOException e) {
470                                                 DavGatewayTray.log(e);
471                                                 LOGGER.warn("Ignore broken message " + rangeIterator.currentIndex + ' ' + e.getMessage());
472                                             }
473 
474                                         }
475                                         sendClient(commandId + " OK FETCH completed");
476                                     }
477 
478                                 } else if ("store".equalsIgnoreCase(command)) {
479                                     RangeIterator rangeIterator = new RangeIterator(currentFolder.messages, tokens.nextToken());
480                                     String action = tokens.nextToken();
481                                     String flags = tokens.nextToken();
482                                     handleStore(commandId, rangeIterator, action, flags);
483 
484                                 } else if ("copy".equalsIgnoreCase(command) || "move".equalsIgnoreCase(command)) {
485                                     try {
486                                         RangeIterator rangeIterator = new RangeIterator(currentFolder.messages, tokens.nextToken());
487                                         String targetName = buildFolderPath(tokens.nextToken());
488                                         if (!rangeIterator.hasNext()) {
489                                             sendClient(commandId + " NO " + "No message found");
490                                         } else {
491                                             while (rangeIterator.hasNext()) {
492                                                 DavGatewayTray.switchIcon();
493                                                 ExchangeSession.Message message = rangeIterator.next();
494                                                 if ("copy".equalsIgnoreCase(command)) {
495                                                     session.copyMessage(message, targetName);
496                                                 } else {
497                                                     session.moveMessage(message, targetName);
498                                                 }
499                                             }
500                                             if ("move".equalsIgnoreCase(command)) {
501                                                 refreshCurrentFolder();
502                                             }
503                                             sendClient(commandId + " OK " + command + " completed");
504                                         }
505                                     } catch (HttpResponseException e) {
506                                         sendClient(commandId + " NO " + e.getMessage());
507                                     }
508                                 } else if ("append".equalsIgnoreCase(command)) {
509                                     String folderName = buildFolderPath(tokens.nextToken());
510                                     HashMap<String, String> properties = new HashMap<>();
511                                     String flags = null;
512                                     String date = null;
513                                     // handle optional flags
514                                     String nextToken = tokens.nextQuotedToken();
515                                     if (nextToken.startsWith("(")) {
516                                         flags = StringUtil.removeQuotes(nextToken);
517                                         if (tokens.hasMoreTokens()) {
518                                             nextToken = tokens.nextToken();
519                                             if (tokens.hasMoreTokens()) {
520                                                 date = nextToken;
521                                                 nextToken = tokens.nextToken();
522                                             }
523                                         }
524                                     } else if (tokens.hasMoreTokens()) {
525                                         date = StringUtil.removeQuotes(nextToken);
526                                         nextToken = tokens.nextToken();
527                                     }
528 
529                                     if (flags != null) {
530                                         HashSet<String> keywords = null;
531                                         // parse flags, on create read and draft flags are on the
532                                         // same messageFlags property, 8 means draft and 1 means read
533                                         ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
534                                         while (flagtokenizer.hasMoreTokens()) {
535                                             String flag = flagtokenizer.nextToken();
536                                             if ("\\Seen".equalsIgnoreCase(flag)) {
537                                                 if (properties.containsKey("draft")) {
538                                                     // draft message, add read flag
539                                                     properties.put("draft", "9");
540                                                 } else {
541                                                     // not (yet) draft, set read flag
542                                                     properties.put("draft", "1");
543                                                 }
544                                             } else if ("\\Flagged".equalsIgnoreCase(flag)) {
545                                                 properties.put("flagged", "2");
546                                             } else if ("\\Answered".equalsIgnoreCase(flag)) {
547                                                 properties.put("answered", "102");
548                                             } else if ("$Forwarded".equalsIgnoreCase(flag)) {
549                                                 properties.put("forwarded", "104");
550                                             } else if ("\\Draft".equalsIgnoreCase(flag)) {
551                                                 if (properties.containsKey("draft")) {
552                                                     // read message, add draft flag
553                                                     properties.put("draft", "9");
554                                                 } else {
555                                                     // not (yet) read, set draft flag
556                                                     properties.put("draft", "8");
557                                                 }
558                                             } else if ("Junk".equalsIgnoreCase(flag)) {
559                                                 properties.put("junk", "1");
560                                             } else {
561                                                 if (keywords == null) {
562                                                     keywords = new HashSet<>();
563                                                 }
564                                                 keywords.add(flag);
565                                             }
566                                         }
567                                         if (keywords != null) {
568                                             properties.put("keywords", session.convertFlagsToKeywords(keywords));
569                                         }
570                                     } else {
571                                         // no flags, force not draft and unread
572                                         properties.put("draft", "0");
573                                     }
574                                     // handle optional date
575                                     if (date != null) {
576                                         SimpleDateFormat dateParser = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss Z", Locale.ENGLISH);
577                                         Date dateReceived = dateParser.parse(date);
578                                         SimpleDateFormat dateFormatter = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
579                                         dateFormatter.setTimeZone(ExchangeSession.GMT_TIMEZONE);
580 
581                                         properties.put("datereceived", dateFormatter.format(dateReceived));
582                                     }
583                                     int size = Integer.parseInt(StringUtil.removeQuotes(nextToken));
584                                     sendClient("+ send literal data");
585                                     byte[] buffer = in.readContent(size);
586                                     // empty line
587                                     readClient();
588                                     MimeMessage mimeMessage = new MimeMessage(null, new SharedByteArrayInputStream(buffer));
589 
590                                     String messageName = UUID.randomUUID() + ".EML";
591                                     try {
592                                         ExchangeSession.Message createdMessage = MessageCreateThread.createMessage(session, folderName, messageName, properties, mimeMessage, os, capabilities);
593                                         if (createdMessage != null) {
594                                             long uid = createdMessage.getImapUid();
595                                             sendClient(commandId + " OK [APPENDUID 1 "+uid+"] APPEND completed");
596                                         } else {
597                                             sendClient(commandId + " OK APPEND completed");
598                                         }
599                                     } catch (InsufficientStorageException e) {
600                                         sendClient(commandId + " NO " + e.getMessage());
601                                     }
602                                 } else if ("idle".equalsIgnoreCase(command) && imapIdleDelay > 0) {
603                                     if (currentFolder != null) {
604                                         sendClient("+ idling ");
605                                         // clear cache before going to idle mode
606                                         currentFolder.clearCache();
607                                         DavGatewayTray.resetIcon();
608                                         int originalTimeout = client.getSoTimeout();
609                                         try {
610                                             int count = 0;
611                                             client.setSoTimeout(1000);
612                                             while (in.available() == 0) {
613                                                 if (++count >= imapIdleDelay) {
614                                                     count = 0;
615                                                     refreshCurrentFolder();
616                                                 }
617                                                 // wait for input 1 second
618                                                 try {
619                                                     byte[] byteBuffer = new byte[1];
620                                                     int readResult = in.read(byteBuffer);
621                                                     if (readResult > 0) {
622                                                         in.unread(byteBuffer);
623                                                     } else if (readResult == -1) {
624                                                         // client disconnected
625                                                         throw new SocketException("client closed connection");
626                                                     }
627                                                 } catch (SocketTimeoutException e) {
628                                                     // ignore, read timed out
629                                                 }
630                                             }
631                                             // read DONE line
632                                             line = readClient();
633                                             if ("DONE".equals(line)) {
634                                                 sendClient(commandId + " OK " + command + " terminated");
635                                             } else {
636                                                 sendClient(commandId + " BAD command unrecognized");
637                                             }
638                                         } catch (IOException e) {
639                                             // client connection closed
640                                             throw new SocketException(e.getMessage());
641                                         } finally {
642                                             client.setSoTimeout(originalTimeout);
643                                         }
644                                     } else {
645                                         sendClient(commandId + " NO no folder selected");
646                                     }
647                                 } else if ("noop".equalsIgnoreCase(command) || "check".equalsIgnoreCase(command)) {
648                                     if (currentFolder != null) {
649                                         DavGatewayTray.debug(new BundleMessage("LOG_IMAP_COMMAND", command, currentFolder.folderPath));
650                                         refreshCurrentFolder();
651                                     }
652                                     sendClient(commandId + " OK " + command + " completed");
653                                 } else if ("subscribe".equalsIgnoreCase(command) || "unsubscribe".equalsIgnoreCase(command)) {
654                                     sendClient(commandId + " OK " + command + " completed");
655                                 } else if ("status".equalsIgnoreCase(command)) {
656                                     try {
657                                         String encodedFolderName = tokens.nextToken();
658                                         ExchangeSession.Folder folder = session.getFolder(buildFolderPath(encodedFolderName));
659                                         // must retrieve messages
660 
661                                         loadFolder(folder);
662 
663                                         String parameters = tokens.nextToken();
664                                         StringBuilder answer = new StringBuilder();
665                                         ImapTokenizer parametersTokens = new ImapTokenizer(parameters);
666                                         while (parametersTokens.hasMoreTokens()) {
667                                             String token = parametersTokens.nextToken();
668                                             if ("MESSAGES".equalsIgnoreCase(token)) {
669                                                 answer.append("MESSAGES ").append(folder.count()).append(' ');
670                                             }
671                                             if ("RECENT".equalsIgnoreCase(token)) {
672                                                 answer.append("RECENT ").append(folder.recent).append(' ');
673                                             }
674                                             if ("UIDNEXT".equalsIgnoreCase(token)) {
675                                                 if (folder.count() == 0) {
676                                                     answer.append("UIDNEXT 1 ");
677                                                 } else {
678                                                     if (folder.count() == 0) {
679                                                         answer.append("UIDNEXT 1 ");
680                                                     } else {
681                                                         answer.append("UIDNEXT ").append(folder.getUidNext()).append(' ');
682                                                     }
683                                                 }
684 
685                                             }
686                                             if ("UIDVALIDITY".equalsIgnoreCase(token)) {
687                                                 answer.append("UIDVALIDITY 1 ");
688                                             }
689                                             if ("UNSEEN".equalsIgnoreCase(token)) {
690                                                 answer.append("UNSEEN ").append(folder.unreadCount).append(' ');
691                                             }
692                                         }
693                                         sendClient("* STATUS \"" + encodedFolderName + "\" (" + answer.toString().trim() + ')');
694                                         sendClient(commandId + " OK " + command + " completed");
695                                     } catch (HttpResponseException e) {
696                                         sendClient(commandId + " NO folder not found");
697                                     }
698                                 } else {
699                                     sendClient(commandId + " BAD command unrecognized");
700                                 }
701                             }
702                         }
703 
704                     } else {
705                         sendClient(commandId + " BAD missing command");
706                     }
707                 } else {
708                     sendClient("BAD Null command");
709                 }
710                 DavGatewayTray.resetIcon();
711             }
712 
713             os.flush();
714         } catch (SocketTimeoutException e) {
715             DavGatewayTray.debug(new BundleMessage("LOG_CLOSE_CONNECTION_ON_TIMEOUT"));
716             try {
717                 sendClient("* BYE Closing connection");
718             } catch (IOException e1) {
719                 DavGatewayTray.debug(new BundleMessage("LOG_EXCEPTION_CLOSING_CONNECTION_ON_TIMEOUT"));
720             }
721         } catch (SocketException e) {
722             LOGGER.warn(BundleMessage.formatLog("LOG_CLIENT_CLOSED_CONNECTION"));
723         } catch (NetworkDownException | UnknownHostException | SSLException e) {
724             LOGGER.warn(e.getMessage());
725             try {
726                 sendClient("* BYE [NetworkDown] " + ((e.getMessage() == null) ? e.toString() : e.getMessage()).replaceAll("\\n", " "));
727             } catch (IOException e2) {
728                 DavGatewayTray.warn(new BundleMessage("LOG_EXCEPTION_SENDING_ERROR_TO_CLIENT"), e2);
729             }
730         } catch (Exception e) {
731             DavGatewayTray.log(e);
732             try {
733                 String message = ((e.getMessage() == null) ? e.toString() : e.getMessage()).replaceAll("\\n", " ");
734                 if (commandId != null) {
735                     sendClient(commandId + " BAD unable to handle request: " + message);
736                 } else {
737                     sendClient("* BAD unable to handle request: " + message);
738                 }
739             } catch (IOException e2) {
740                 DavGatewayTray.warn(new BundleMessage("LOG_EXCEPTION_SENDING_ERROR_TO_CLIENT"), e2);
741             }
742         } finally {
743             close();
744         }
745         DavGatewayTray.resetIcon();
746     }
747 
748     protected String lastCommand;
749     protected int lastCommandCount;
750 
751     /**
752      * Detect infinite loop on the client side.
753      *
754      * @param line IMAP command line
755      * @throws IOException on infinite loop
756      */
757     protected void checkInfiniteLoop(String line) throws IOException {
758         int spaceIndex = line.indexOf(' ');
759         if (spaceIndex < 0) {
760             // invalid command line, reset
761             lastCommand = null;
762             lastCommandCount = 0;
763         } else {
764             String command = line.substring(spaceIndex + 1);
765             if (command.equals(lastCommand)) {
766                 lastCommandCount++;
767                 if (lastCommandCount > 100 && !"NOOP".equalsIgnoreCase(lastCommand) && !"IDLE".equalsIgnoreCase(lastCommand)) {
768                     // more than a hundred times the same command => this is a client infinite loop, close connection
769                     throw new IOException("Infinite loop on command " + command + " detected");
770                 }
771             } else {
772                 // new command, reset
773                 lastCommand = command;
774                 lastCommandCount = 0;
775             }
776         }
777     }
778 
779     /**
780      * Detect shared mailbox access.
781      * see <a href="https://help.ubuntu.com/community/ThunderbirdExchange">Connecting to a Microsoft Exchange Server with Thunderbird</a>
782      */
783     protected void splitUserName() {
784         String[] tokens = null;
785         if (userName.indexOf('/') >= 0) {
786             tokens = userName.split("/");
787         } else if (userName.indexOf('\\') >= 0) {
788             tokens = userName.split("\\\\");
789         }
790 
791         if (tokens != null && tokens.length == 3) {
792             userName = tokens[0] + '\\' + tokens[1];
793             baseMailboxPath = "/users/" + tokens[2] + '/';
794         } else if (tokens != null && tokens.length == 2 && tokens[1].contains("@")) {
795             userName = tokens[0];
796             baseMailboxPath = "/users/" + tokens[1] + '/';
797         }
798     }
799 
800     protected String encodeFolderPath(String folderPath) {
801         return BASE64MailboxEncoder.encode(folderPath).replaceAll("\"", "\\\\\"");
802     }
803 
804     protected String decodeFolderPath(String folderPath) {
805         return BASE64MailboxDecoder.decode(folderPath)
806                 //unescape quotes inside value
807                 .replaceAll("\\\\", "");
808     }
809 
810     protected String buildFolderPath(String encodedFolderPath) {
811         String decodedFolderPath = decodeFolderPath(encodedFolderPath);
812         if (baseMailboxPath == null || decodedFolderPath.startsWith("/")) {
813             return decodedFolderPath;
814         } else {
815             return baseMailboxPath + decodedFolderPath;
816         }
817     }
818 
819     /**
820      * Send expunge untagged response for removed IMAP message uids.
821      *
822      * @param previousImapFlagMap uid map before refresh
823      * @param imapFlagMap         uid map after refresh
824      * @throws IOException on error
825      */
826     private void handleRefresh(TreeMap<Long, String> previousImapFlagMap, TreeMap<Long, String> imapFlagMap) throws IOException {
827         // send deleted message expunge notification
828         int index = 1;
829         for (long previousImapUid : previousImapFlagMap.keySet()) {
830             if (!imapFlagMap.containsKey(previousImapUid)) {
831                 sendClient("* " + index + " EXPUNGE");
832             } else {
833                 // send updated flags
834                 if (!previousImapFlagMap.get(previousImapUid).equals(imapFlagMap.get(previousImapUid))) {
835                     sendClient("* " + index + " FETCH (UID " + previousImapUid + " FLAGS (" + imapFlagMap.get(previousImapUid) + "))");
836                 }
837                 index++;
838             }
839         }
840 
841         sendClient("* " + currentFolder.count() + " EXISTS");
842         sendClient("* " + currentFolder.recent + " RECENT");
843     }
844 
845     /**
846      * Refresh current folder and send EXPUNGE/FLAG notifications.
847      *
848      * @throws IOException on error
849      */
850     private void refreshCurrentFolder() throws IOException {
851         TreeMap<Long, String> previousImapFlagMap = currentFolder.getImapFlagMap();
852         if (session.refreshFolder(currentFolder)) {
853             handleRefresh(previousImapFlagMap, currentFolder.getImapFlagMap());
854         }
855     }
856 
857     static private class KeepAlive {
858         private final ScheduledExecutorService scheduler;
859         private final ScheduledFuture<?> task;
860 
861         public KeepAlive(Runnable cb) {
862             scheduler = Executors.newScheduledThreadPool(1, r -> new Thread(r, currentThread().getName() + "-KeepAlive"));
863             task = scheduler.scheduleAtFixedRate(cb, 20, 20, TimeUnit.SECONDS);
864         }
865 
866         public void cancel() {
867             task.cancel(true);
868             scheduler.shutdown();
869         }
870     }
871 
872     private void loadFolder(ExchangeSession.Folder folder) throws IOException {
873         final boolean enable = Settings.getBooleanProperty("davmail.enableKeepAlive", false);
874         KeepAlive keepAlive = null;
875 
876         if (enable && (folder.count() > 500)) {
877             Runnable progressNotifier = () -> {
878                 try {
879                     sendClient("* OK in progress");
880                 } catch (IOException e) {
881                     LOGGER.warn("Error while sending progress notification", e);
882                 }
883             };
884             keepAlive = new KeepAlive(progressNotifier);
885         }
886 
887         try {
888             folder.loadMessages();
889         } finally {
890             if (keepAlive != null) {
891                 keepAlive.cancel();
892             }
893         }
894     }
895 
896     static protected class MessageWrapper {
897         protected OutputStream os;
898         protected StringBuilder buffer;
899         protected ExchangeSession.Message message;
900 
901         protected MessageWrapper(OutputStream os, StringBuilder buffer, ExchangeSession.Message message) {
902             this.os = os;
903             this.buffer = buffer;
904             this.message = message;
905         }
906 
907         public int getMimeMessageSize() throws IOException, MessagingException {
908             loadMessage();
909             return message.getMimeMessageSize();
910         }
911 
912         /**
913          * Monitor full message download
914          */
915         protected void loadMessage() throws IOException, MessagingException {
916             if (!message.isLoaded()) {
917                 // flush current buffer
918                 String flushString = buffer.toString();
919                 LOGGER.debug(flushString);
920                 os.write(flushString.getBytes(StandardCharsets.UTF_8));
921                 buffer.setLength(0);
922                 MessageLoadThread.loadMimeMessage(message, os);
923             }
924         }
925 
926         public MimeMessage getMimeMessage() throws IOException, MessagingException {
927             loadMessage();
928             return message.getMimeMessage();
929         }
930 
931         public InputStream getRawInputStream() throws IOException, MessagingException {
932             loadMessage();
933             return message.getRawInputStream();
934         }
935 
936         public Enumeration<?> getMatchingHeaderLines(String[] requestedHeaders) throws IOException, MessagingException {
937             Enumeration<?> result = message.getMatchingHeaderLinesFromHeaders(requestedHeaders);
938             if (result == null) {
939                 loadMessage();
940                 result = message.getMatchingHeaderLines(requestedHeaders);
941             }
942             return result;
943         }
944     }
945 
946 
947     private void handleFetch(ExchangeSession.Message message, int currentIndex, String parameters) throws IOException, MessagingException {
948         StringBuilder buffer = new StringBuilder();
949         MessageWrapper messageWrapper = new MessageWrapper(os, buffer, message);
950         buffer.append("* ").append(currentIndex).append(" FETCH (UID ").append(message.getImapUid());
951         if (parameters != null) {
952             parameters = handleFetchMacro(parameters);
953             ImapTokenizer paramTokens = new ImapTokenizer(parameters);
954             while (paramTokens.hasMoreTokens()) {
955                 @SuppressWarnings({"NonConstantStringShouldBeStringBuffer"})
956                 String param = paramTokens.nextToken().toUpperCase();
957                 if ("FLAGS".equals(param)) {
958                     buffer.append(" FLAGS (").append(message.getImapFlags()).append(')');
959                 } else if ("RFC822.SIZE".equals(param)) {
960                     int size;
961                     if ((parameters.contains("BODY.PEEK[HEADER.FIELDS (")
962                             // exclude mutt header request
963                             && !parameters.contains("X-LABEL"))
964                             || parameters.equals("RFC822.SIZE RFC822.HEADER FLAGS") // icedove
965                             || Settings.getBooleanProperty("davmail.imapAlwaysApproxMsgSize")) {   // Send approximate size
966                         size = message.size;
967                         LOGGER.debug(String.format("Message %s in folder %s sent approximate size %d bytes", message.getImapUid(), currentFolder.folderPath, size));
968                     } else {
969                         size = messageWrapper.getMimeMessageSize();
970                     }
971                     buffer.append(" RFC822.SIZE ").append(size);
972                 } else if ("ENVELOPE".equals(param)) {
973                     appendEnvelope(buffer, messageWrapper);
974                 } else if ("BODYSTRUCTURE".equals(param)) {
975                     appendBodyStructure(buffer, messageWrapper);
976                 } else if ("INTERNALDATE".equals(param) && message.date != null && !message.date.isEmpty()) {
977                     try {
978                         SimpleDateFormat dateParser = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss.SSS'Z'");
979                         dateParser.setTimeZone(ExchangeSession.GMT_TIMEZONE);
980                         Date date = ExchangeSession.getZuluDateFormat().parse(message.date);
981                         SimpleDateFormat dateFormatter = new SimpleDateFormat("dd-MMM-yyyy HH:mm:ss Z", Locale.ENGLISH);
982                         buffer.append(" INTERNALDATE \"").append(dateFormatter.format(date)).append('\"');
983                     } catch (ParseException e) {
984                         throw new DavMailException("EXCEPTION_INVALID_DATE", message.date);
985                     }
986                 } else if ("RFC822".equals(param) || param.startsWith("BODY[") || param.startsWith("BODY.PEEK[")
987                         || "RFC822.HEADER".equals(param) || "RFC822.TEXT".equals(param)) {
988 
989                     if (param.startsWith("BODY[") && !message.read) {
990                         // According to IMAP RFC: The \Seen flag is implicitly set
991                         updateFlags(message, "FLAGS", "\\Seen");
992                         message.read = true;
993                     }
994 
995                     // get full param
996                     if (param.indexOf('[') >= 0) {
997                         StringBuilder paramBuffer = new StringBuilder(param);
998                         while (paramTokens.hasMoreTokens() && paramBuffer.indexOf("]") < 0) {
999                             paramBuffer.append(' ').append(paramTokens.nextToken());
1000                         }
1001                         param = paramBuffer.toString();
1002                     }
1003                     // parse buffer size
1004                     int startIndex = 0;
1005                     int maxSize = Integer.MAX_VALUE;
1006                     int ltIndex = param.indexOf('<');
1007                     if (ltIndex >= 0) {
1008                         int dotIndex = param.indexOf('.', ltIndex);
1009                         if (dotIndex >= 0) {
1010                             startIndex = Integer.parseInt(param.substring(ltIndex + 1, dotIndex));
1011                             maxSize = Integer.parseInt(param.substring(dotIndex + 1, param.indexOf('>')));
1012                         }
1013                     }
1014 
1015                     ByteArrayOutputStream baos = new ByteArrayOutputStream();
1016                     InputStream partInputStream = null;
1017                     OutputStream partOutputStream = null;
1018 
1019                     // try to parse message part index
1020                     String partIndexString = StringUtil.getToken(param, "[", "]");
1021                     if ((partIndexString == null || partIndexString.isEmpty()) && !"RFC822.HEADER".equals(param)) {
1022                         // write message with headers
1023                         partOutputStream = new PartialOutputStream(baos, startIndex, maxSize);
1024                         partInputStream = messageWrapper.getRawInputStream();
1025                     } else if ("TEXT".equals(partIndexString)) {
1026                         // write message without headers
1027                         partOutputStream = new PartOutputStream(baos, false, true, startIndex, maxSize);
1028                         partInputStream = messageWrapper.getRawInputStream();
1029                     } else if ("RFC822.HEADER".equals(param) || (partIndexString != null && partIndexString.startsWith("HEADER"))) {
1030                         // Header requested fetch headers
1031                         String[] requestedHeaders = getRequestedHeaders(partIndexString);
1032                         // OSX Lion special flags request
1033                         if (requestedHeaders != null && requestedHeaders.length == 1 && "content-class".equals(requestedHeaders[0]) && message.contentClass != null) {
1034                             baos.write("Content-class: ".getBytes(StandardCharsets.UTF_8));
1035                             baos.write(message.contentClass.getBytes(StandardCharsets.UTF_8));
1036                             baos.write(13);
1037                             baos.write(10);
1038                         } else if (requestedHeaders == null) {
1039                             // load message and write all headers
1040                             partOutputStream = new PartOutputStream(baos, true, false, startIndex, maxSize);
1041                             partInputStream = messageWrapper.getRawInputStream();
1042                         } else {
1043                             Enumeration<?> headerEnumeration = messageWrapper.getMatchingHeaderLines(requestedHeaders);
1044                             while (headerEnumeration.hasMoreElements()) {
1045                                 baos.write(((String) headerEnumeration.nextElement()).getBytes(StandardCharsets.UTF_8));
1046                                 baos.write(13);
1047                                 baos.write(10);
1048                             }
1049                         }
1050                     } else if (partIndexString != null) {
1051                         MimePart bodyPart = messageWrapper.getMimeMessage();
1052                         String[] partIndexStrings = partIndexString.split("\\.");
1053                         for (String subPartIndexString : partIndexStrings) {
1054                             // ignore MIME subpart index, will return full part
1055                             if ("MIME".equals(subPartIndexString)) {
1056                                 break;
1057                             }
1058                             int subPartIndex;
1059                             // try to parse part index
1060                             try {
1061                                 subPartIndex = Integer.parseInt(subPartIndexString);
1062                             } catch (NumberFormatException e) {
1063                                 throw new DavMailException("EXCEPTION_INVALID_PARAMETER", param);
1064                             }
1065 
1066                             Object mimeBody = bodyPart.getContent();
1067                             if (mimeBody instanceof MimeMultipart) {
1068                                 MimeMultipart multiPart = (MimeMultipart) mimeBody;
1069                                 if (subPartIndex - 1 < multiPart.getCount()) {
1070                                     bodyPart = (MimePart) multiPart.getBodyPart(subPartIndex - 1);
1071                                 } else {
1072                                     throw new DavMailException("EXCEPTION_INVALID_PARAMETER", param);
1073                                 }
1074                             } else if (subPartIndex != 1) {
1075                                 throw new DavMailException("EXCEPTION_INVALID_PARAMETER", param);
1076                             }
1077                         }
1078 
1079                         // write selected part, without headers
1080                         partOutputStream = new PartialOutputStream(baos, startIndex, maxSize);
1081                         if (bodyPart instanceof MimeMessage) {
1082                             partInputStream = ((MimeMessage) bodyPart).getRawInputStream();
1083                         } else {
1084                             partInputStream = ((MimeBodyPart) bodyPart).getRawInputStream();
1085                         }
1086                     }
1087 
1088                     // copy selected content to baos
1089                     if (partInputStream != null && partOutputStream != null) {
1090                         IOUtil.write(partInputStream, partOutputStream);
1091                         partInputStream.close();
1092                         partOutputStream.close();
1093                     }
1094                     baos.close();
1095 
1096                     if ("RFC822".equals(param)) {
1097                         buffer.append(" RFC822");
1098                     } else if ("RFC822.HEADER".equals(param)) {
1099                         buffer.append(" RFC822.HEADER");
1100                     } else if ("RFC822.TEXT".equals(param)) {
1101                         buffer.append(" RFC822.TEXT");
1102                     } else {
1103                         buffer.append(" BODY[");
1104                         if (partIndexString != null) {
1105                             buffer.append(partIndexString);
1106                         }
1107                         buffer.append(']');
1108                     }
1109                     // partial
1110                     if (startIndex > 0 || maxSize != Integer.MAX_VALUE) {
1111                         buffer.append('<').append(startIndex).append('>');
1112                     }
1113                     buffer.append(" {").append(baos.size()).append('}');
1114                     sendClient(buffer.toString());
1115                     os.write(baos.toByteArray());
1116                     os.flush();
1117                     buffer.setLength(0);
1118                 }
1119             }
1120         }
1121         buffer.append(')');
1122         sendClient(buffer.toString());
1123         // do not keep message content in memory
1124         message.dropMimeMessage();
1125     }
1126 
1127     /**
1128      * Handle flags macro in fetch requests
1129      * @param parameters input fetch flags
1130      * @return transformed fetch flags
1131      */
1132     private String handleFetchMacro(String parameters) {
1133         if ("ALL".equals(parameters)) {
1134             return "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE";
1135         } else if ("FAST".equals(parameters)) {
1136             return "FLAGS INTERNALDATE RFC822.SIZE";
1137         } else if ("FULL".equals(parameters)) {
1138             return "FLAGS INTERNALDATE RFC822.SIZE ENVELOPE BODY";
1139         } else {
1140             return parameters;
1141         }
1142     }
1143 
1144     protected String[] getRequestedHeaders(String partIndexString) {
1145         if (partIndexString == null) {
1146             return null;
1147         } else {
1148             int startIndex = partIndexString.indexOf('(');
1149             int endIndex = partIndexString.indexOf(')');
1150             if (startIndex >= 0 && endIndex >= 0) {
1151                 return partIndexString.substring(startIndex + 1, endIndex).split(" ");
1152             } else {
1153                 return null;
1154             }
1155         }
1156     }
1157 
1158     protected void handleStore(String commandId, AbstractRangeIterator rangeIterator, String action, String flags) throws IOException {
1159         while (rangeIterator.hasNext()) {
1160             DavGatewayTray.switchIcon();
1161             ExchangeSession.Message message = rangeIterator.next();
1162             updateFlags(message, action, flags);
1163             sendClient("* " + (rangeIterator.getCurrentIndex()) + " FETCH (UID " + message.getImapUid() + " FLAGS (" + (message.getImapFlags()) + "))");
1164         }
1165         // auto expunge
1166         if (Settings.getBooleanProperty("davmail.imapAutoExpunge")) {
1167             if (expunge(false)) {
1168                 session.refreshFolder(currentFolder);
1169             }
1170         }
1171         sendClient(commandId + " OK STORE completed");
1172     }
1173 
1174     protected ExchangeSession.Condition buildConditions(SearchConditions conditions, ImapTokenizer tokens) throws IOException {
1175         ExchangeSession.MultiCondition condition = null;
1176         while (tokens.hasMoreTokens()) {
1177             String token = tokens.nextQuotedToken().toUpperCase();
1178             if (token.startsWith("(") && token.endsWith(")")) {
1179                 // quoted search param
1180                 if (condition == null) {
1181                     condition = session.and();
1182                 }
1183                 condition.add(buildConditions(conditions, new ImapTokenizer(token.substring(1, token.length() - 1))));
1184             } else if ("OR".equals(token)) {
1185                 condition = session.or();
1186             } else if (token.startsWith("OR ")) {
1187                 condition = appendOrSearchParams(token, conditions);
1188             } else if ("CHARSET".equals(token)) {
1189                 String charset = tokens.nextToken().toUpperCase();
1190                 if (!("ASCII".equals(charset) || "UTF-8".equals(charset) || "US-ASCII".equals(charset))) {
1191                     throw new IOException("Unsupported charset " + charset);
1192                 }
1193             } else {
1194                 if (condition == null) {
1195                     condition = session.and();
1196                 }
1197                 condition.add(appendSearchParam(tokens, token, conditions));
1198             }
1199         }
1200         return condition;
1201     }
1202 
1203 
1204     protected List<Long> handleSearch(ImapTokenizer tokens) throws IOException {
1205         List<Long> uidList = new ArrayList<>();
1206         List<Long> localMessagesUidList = null;
1207         SearchConditions conditions = new SearchConditions();
1208         ExchangeSession.Condition condition = buildConditions(conditions, tokens);
1209         session.refreshFolder(currentFolder);
1210         ExchangeSession.MessageList localMessages = currentFolder.searchMessages(condition);
1211         Iterator<ExchangeSession.Message> iterator;
1212         if (conditions.uidRange != null) {
1213             iterator = new UIDRangeIterator(localMessages, conditions.uidRange);
1214         } else if (conditions.indexRange != null) {
1215             // range iterator is on folder messages, not messages returned from search
1216             iterator = new RangeIterator(currentFolder.messages, conditions.indexRange);
1217             localMessagesUidList = new ArrayList<>();
1218             // build search result uid list
1219             for (ExchangeSession.Message message : localMessages) {
1220                 localMessagesUidList.add(message.getImapUid());
1221             }
1222         } else {
1223             iterator = localMessages.iterator();
1224         }
1225         while (iterator.hasNext()) {
1226             ExchangeSession.Message message = iterator.next();
1227             if ((conditions.flagged == null || message.flagged == conditions.flagged)
1228                     && (conditions.answered == null || message.answered == conditions.answered)
1229                     && (conditions.draft == null || message.draft == conditions.draft)
1230                     // range iterator: include messages available in search result
1231                     && (localMessagesUidList == null || localMessagesUidList.contains(message.getImapUid()))
1232                     && isNotExcluded(conditions.notUidRange, message.getImapUid())) {
1233                 uidList.add(message.getImapUid());
1234             }
1235         }
1236         return uidList;
1237     }
1238 
1239     /**
1240      * Check NOT UID condition.
1241      *
1242      * @param notUidRange excluded uid range
1243      * @param imapUid     current message imap uid
1244      * @return true if not excluded
1245      */
1246     private boolean isNotExcluded(String notUidRange, long imapUid) {
1247         if (notUidRange == null) {
1248             return true;
1249         }
1250         String imapUidAsString = String.valueOf(imapUid);
1251         for (String rangeValue : notUidRange.split(",")) {
1252             if (imapUidAsString.equals(rangeValue)) {
1253                 return false;
1254             }
1255         }
1256         return true;
1257     }
1258 
1259     protected void appendEnvelope(StringBuilder buffer, MessageWrapper message) throws IOException {
1260 
1261         try {
1262             MimeMessage mimeMessage = message.getMimeMessage();
1263             buffer.append(" ENVELOPE ");
1264             appendEnvelope(buffer, mimeMessage);
1265         } catch (MessagingException me) {
1266             DavGatewayTray.warn(me);
1267             // send fake envelope
1268             buffer.append(" ENVELOPE (NIL NIL NIL NIL NIL NIL NIL NIL NIL NIL)");
1269         }
1270     }
1271 
1272     private void appendEnvelope(StringBuilder buffer, MimePart mimePart) throws UnsupportedEncodingException, MessagingException {
1273         buffer.append('(');
1274         // Envelope for date, subject, from, sender, reply-to, to, cc, bcc,in-reply-to, message-id
1275         appendEnvelopeHeader(buffer, mimePart.getHeader("Date"));
1276         appendEnvelopeHeader(buffer, mimePart.getHeader("Subject"));
1277         appendMailEnvelopeHeader(buffer, mimePart.getHeader("From"));
1278         appendMailEnvelopeHeader(buffer, mimePart.getHeader("Sender"));
1279         appendMailEnvelopeHeader(buffer, mimePart.getHeader("Reply-To"));
1280         appendMailEnvelopeHeader(buffer, mimePart.getHeader("To"));
1281         appendMailEnvelopeHeader(buffer, mimePart.getHeader("CC"));
1282         appendMailEnvelopeHeader(buffer, mimePart.getHeader("BCC"));
1283         appendEnvelopeHeader(buffer, mimePart.getHeader("In-Reply-To"));
1284         appendEnvelopeHeader(buffer, mimePart.getHeader("Message-Id"));
1285         buffer.append(')');
1286     }
1287 
1288     protected void appendEnvelopeHeader(StringBuilder buffer, String[] value) throws UnsupportedEncodingException {
1289         if (buffer.charAt(buffer.length() - 1) != '(') {
1290             buffer.append(' ');
1291         }
1292         if (value != null && value.length > 0) {
1293             appendEnvelopeHeaderValue(buffer, MimeUtility.unfold(value[0]));
1294         } else {
1295             buffer.append("NIL");
1296         }
1297     }
1298 
1299     protected void appendMailEnvelopeHeader(StringBuilder buffer, String[] value) {
1300         buffer.append(' ');
1301         if (value != null && value.length > 0) {
1302             try {
1303                 String unfoldedValue = MimeUtility.unfold(value[0]);
1304                 InternetAddress[] addresses = InternetAddress.parseHeader(unfoldedValue, false);
1305                 if (addresses.length > 0) {
1306                     buffer.append('(');
1307                     for (InternetAddress address : addresses) {
1308                         buffer.append('(');
1309                         String personal = address.getPersonal();
1310                         if (personal != null) {
1311                             appendEnvelopeHeaderValue(buffer, personal);
1312                         } else {
1313                             buffer.append("NIL");
1314                         }
1315                         buffer.append(" NIL ");
1316                         String mail = address.getAddress();
1317                         int atIndex = mail.indexOf('@');
1318                         if (atIndex >= 0) {
1319                             buffer.append('"').append(mail, 0, atIndex).append('"');
1320                             buffer.append(' ');
1321                             buffer.append('"').append(mail.substring(atIndex + 1)).append('"');
1322                         } else {
1323                             buffer.append("NIL NIL");
1324                         }
1325                         buffer.append(')');
1326                     }
1327                     buffer.append(')');
1328                 } else {
1329                     buffer.append("NIL");
1330                 }
1331             } catch (AddressException | UnsupportedEncodingException e) {
1332                 DavGatewayTray.warn(e);
1333                 buffer.append("NIL");
1334             }
1335         } else {
1336             buffer.append("NIL");
1337         }
1338     }
1339 
1340     /**
1341      * Append envelope header value.
1342      * @param buffer envelope buffer
1343      * @param value header value
1344      * @throws UnsupportedEncodingException on error
1345      */
1346     protected void appendEnvelopeHeaderValue(StringBuilder buffer, String value) throws UnsupportedEncodingException {
1347         String encoded = MimeUtility.encodeText(value, "UTF-8", null);
1348             buffer.append('"');
1349             buffer.append(encoded.replace("\\", "\\\\").replace("\"", "\\\""));
1350             buffer.append('"');
1351     }
1352 
1353     protected void appendBodyStructure(StringBuilder buffer, MessageWrapper message) throws IOException {
1354 
1355         buffer.append(" BODYSTRUCTURE ");
1356         try {
1357             MimeMessage mimeMessage = message.getMimeMessage();
1358             Object mimeBody = mimeMessage.getContent();
1359             if (mimeBody instanceof MimeMultipart) {
1360                 appendBodyStructure(buffer, (MimeMultipart) mimeBody);
1361             } else {
1362                 // no multipart, single body
1363                 appendBodyStructure(buffer, mimeMessage);
1364             }
1365         } catch (UnsupportedEncodingException | MessagingException e) {
1366             DavGatewayTray.warn(e);
1367             // failover: send default bodystructure
1368             buffer.append("(\"TEXT\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 0 0)");
1369         }
1370     }
1371 
1372     protected void appendBodyStructure(StringBuilder buffer, MimeMultipart multiPart) throws IOException, MessagingException {
1373         buffer.append('(');
1374 
1375         for (int i = 0; i < multiPart.getCount(); i++) {
1376             MimeBodyPart bodyPart = (MimeBodyPart) multiPart.getBodyPart(i);
1377             try {
1378                 Object mimeBody = bodyPart.getContent();
1379                 if (mimeBody instanceof MimeMultipart) {
1380                     appendBodyStructure(buffer, (MimeMultipart) mimeBody);
1381                 } else {
1382                     // no multipart, single body
1383                     appendBodyStructure(buffer, bodyPart);
1384                 }
1385             } catch (UnsupportedEncodingException e) {
1386                 LOGGER.warn(e);
1387                 // failover: send default bodystructure
1388                 buffer.append("(\"TEXT\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 0 0)");
1389             } catch (MessagingException me) {
1390                 DavGatewayTray.warn(me);
1391                 // failover: send default bodystructure
1392                 buffer.append("(\"TEXT\" \"PLAIN\" (\"CHARSET\" \"US-ASCII\") NIL NIL \"7BIT\" 0 0)");
1393             }
1394         }
1395         int slashIndex = multiPart.getContentType().indexOf('/');
1396         if (slashIndex < 0) {
1397             throw new DavMailException("EXCEPTION_INVALID_CONTENT_TYPE", multiPart.getContentType());
1398         }
1399         int semiColonIndex = multiPart.getContentType().indexOf(';');
1400         if (semiColonIndex < 0) {
1401             buffer.append(" \"").append(multiPart.getContentType().substring(slashIndex + 1).toUpperCase()).append("\")");
1402         } else {
1403             buffer.append(" \"").append(multiPart.getContentType().substring(slashIndex + 1, semiColonIndex).trim().toUpperCase()).append("\")");
1404         }
1405     }
1406 
1407     protected void appendBodyStructure(StringBuilder buffer, MimePart bodyPart) throws IOException, MessagingException {
1408         String contentType = MimeUtility.unfold(bodyPart.getContentType());
1409         int slashIndex = contentType.indexOf('/');
1410         if (slashIndex < 0) {
1411             throw new DavMailException("EXCEPTION_INVALID_CONTENT_TYPE", contentType);
1412         }
1413         String type = contentType.substring(0, slashIndex).toUpperCase();
1414         buffer.append("(\"").append(type).append("\" \"");
1415         int semiColonIndex = contentType.indexOf(';');
1416         if (semiColonIndex < 0) {
1417             buffer.append(contentType.substring(slashIndex + 1).toUpperCase()).append("\" NIL");
1418         } else {
1419             // extended content type
1420             buffer.append(contentType.substring(slashIndex + 1, semiColonIndex).trim().toUpperCase()).append('\"');
1421             int charsetindex = contentType.indexOf("charset=");
1422             int nameindex = contentType.indexOf("name=");
1423             if (charsetindex >= 0 || nameindex >= 0) {
1424                 buffer.append(" (");
1425 
1426                 if (charsetindex >= 0) {
1427                     buffer.append("\"CHARSET\" ");
1428                     int charsetSemiColonIndex = contentType.indexOf(';', charsetindex);
1429                     int charsetEndIndex;
1430                     if (charsetSemiColonIndex > 0) {
1431                         charsetEndIndex = charsetSemiColonIndex;
1432                     } else {
1433                         charsetEndIndex = contentType.length();
1434                     }
1435                     String charSet = contentType.substring(charsetindex + "charset=".length(), charsetEndIndex);
1436                     if (!charSet.startsWith("\"")) {
1437                         buffer.append('"');
1438                     }
1439                     buffer.append(charSet.trim().toUpperCase());
1440                     if (!charSet.endsWith("\"")) {
1441                         buffer.append('"');
1442                     }
1443                 }
1444 
1445                 if (nameindex >= 0) {
1446                     if (charsetindex >= 0) {
1447                         buffer.append(' ');
1448                     }
1449 
1450                     buffer.append("\"NAME\" ");
1451                     int nameSemiColonIndex = contentType.indexOf(';', nameindex);
1452                     int nameEndIndex;
1453                     if (nameSemiColonIndex > 0) {
1454                         nameEndIndex = nameSemiColonIndex;
1455                     } else {
1456                         nameEndIndex = contentType.length();
1457                     }
1458                     String name = contentType.substring(nameindex + "name=".length(), nameEndIndex).trim();
1459                     if (!name.startsWith("\"")) {
1460                         buffer.append('"');
1461                     }
1462                     buffer.append(name.trim());
1463                     if (!name.endsWith("\"")) {
1464                         buffer.append('"');
1465                     }
1466                 }
1467                 buffer.append(')');
1468             } else {
1469                 buffer.append(" NIL");
1470             }
1471         }
1472         int bodySize = getBodyPartSize(bodyPart);
1473         appendBodyStructureValue(buffer, bodyPart.getContentID());
1474         appendBodyStructureValue(buffer, bodyPart.getDescription());
1475         appendBodyStructureValue(buffer, bodyPart.getEncoding());
1476         appendBodyStructureValue(buffer, bodySize);
1477 
1478         // line count not implemented in JavaMail, return fake line count
1479         int lineCount = bodySize / 80;
1480         if ("TEXT".equals(type)) {
1481             appendBodyStructureValue(buffer, lineCount);
1482         } else if ("MESSAGE".equals(type)) {
1483             Object bodyPartContent = bodyPart.getContent();
1484             if (bodyPartContent instanceof MimeMessage) {
1485                 buffer.append(' '); // RFC 3501: space required before envelope in MESSAGE/RFC822 bodystructure
1486                 MimeMessage innerMessage = (MimeMessage) bodyPartContent;
1487                 appendEnvelope(buffer, innerMessage);
1488                 appendBodyStructure(buffer, innerMessage);
1489                 appendBodyStructureValue(buffer, lineCount);
1490             } else {
1491                 // failover malformed message
1492                 appendBodyStructureValue(buffer, lineCount);
1493             }
1494         }
1495         buffer.append(')');
1496     }
1497 
1498     /**
1499      * Compute body part size with failover.
1500      * @param bodyPart MIME body part
1501      * @return body part size or 0 on error
1502      */
1503     private int getBodyPartSize(MimePart bodyPart) {
1504         int bodySize = 0;
1505         try {
1506             bodySize = bodyPart.getSize();
1507             if (bodySize == -1) {
1508                 // failover, try to get size
1509                 ByteArrayOutputStream baos = new ByteArrayOutputStream();
1510                 bodyPart.writeTo(baos);
1511                 bodySize = baos.size();
1512             }
1513         } catch (IOException | MessagingException e) {
1514             LOGGER.warn("Unable to get body part size " + e.getMessage(), e);
1515         }
1516         return bodySize;
1517     }
1518 
1519     protected void appendBodyStructureValue(StringBuilder buffer, String value) {
1520         if (value == null) {
1521             buffer.append(" NIL");
1522         } else {
1523             buffer.append(" \"").append(value.toUpperCase()).append('\"');
1524         }
1525     }
1526 
1527     protected void appendBodyStructureValue(StringBuilder buffer, int value) {
1528         if (value < 0) {
1529             // use 0 if we don't have a valid number
1530             buffer.append(" 0");
1531         } else {
1532             buffer.append(' ').append(value);
1533         }
1534     }
1535 
1536     protected void sendSubFolders(String command, String folderPath, boolean recursive, boolean wildcard, boolean specialOnly) throws IOException {
1537         try {
1538             List<ExchangeSession.Folder> folders = session.getSubFolders(folderPath, recursive, wildcard);
1539             for (ExchangeSession.Folder folder : folders) {
1540                 if (!specialOnly || folder.isSpecial()) {
1541                     sendClient("* " + command + " (" + folder.getFlags() + ") \"/\" \"" + encodeFolderPath(folder.folderPath) + '\"');
1542                 }
1543             }
1544         } catch (HttpForbiddenException e) {
1545             // access forbidden, ignore
1546             DavGatewayTray.debug(new BundleMessage("LOG_SUBFOLDER_ACCESS_FORBIDDEN", folderPath));
1547         } catch (HttpNotFoundException e) {
1548             // not found, ignore
1549             DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_NOT_FOUND", folderPath));
1550         } catch (HttpResponseException e) {
1551             // other errors, ignore
1552             DavGatewayTray.debug(new BundleMessage("LOG_FOLDER_ACCESS_ERROR", folderPath, e.getMessage()));
1553         }
1554     }
1555 
1556     /**
1557      * client side search conditions
1558      */
1559     static final protected class SearchConditions {
1560         Boolean flagged;
1561         Boolean answered;
1562         Boolean draft;
1563         String indexRange;
1564         String uidRange;
1565         String notUidRange;
1566     }
1567 
1568     protected ExchangeSession.MultiCondition appendOrSearchParams(String token, SearchConditions conditions) throws IOException {
1569         ExchangeSession.MultiCondition orCondition = session.or();
1570         ImapTokenizer innerTokens = new ImapTokenizer(token);
1571         innerTokens.nextToken();
1572         while (innerTokens.hasMoreTokens()) {
1573             String innerToken = innerTokens.nextToken();
1574             orCondition.add(appendSearchParam(innerTokens, innerToken, conditions));
1575         }
1576         return orCondition;
1577     }
1578 
1579     protected ExchangeSession.Condition appendNotSearchParams(String token, SearchConditions conditions) throws IOException {
1580         ImapTokenizer innerTokens = new ImapTokenizer(token);
1581         ExchangeSession.Condition cond = buildConditions(conditions, innerTokens);
1582         if (cond == null || cond.isEmpty()) {
1583             return null;
1584         }
1585         return session.not(cond);
1586     }
1587 
1588     protected ExchangeSession.Condition appendSearchParam(ImapTokenizer tokens, String token, SearchConditions conditions) throws IOException {
1589         if ("NOT".equals(token)) {
1590             String nextToken = tokens.nextToken();
1591             if ("DELETED".equals(nextToken)) {
1592                 // conditions.deleted = Boolean.FALSE;
1593                 return session.isNull("deleted");
1594             } else if ("FROM".equals(nextToken) || "TO".equals(nextToken) || "CC".equals(nextToken)
1595                     || "SUBJECT".equals(nextToken) || "BODY".equals(nextToken) || "TEXT".equals(nextToken)
1596                     || "KEYWORD".equals(nextToken) || "UNKEYWORD".equals(nextToken) || "LARGER".equals(nextToken) || "SMALLER".equals(nextToken)) {
1597                 return appendNotSearchParams(nextToken + " " + tokens.nextToken(), conditions);
1598             } else if ("UID".equals(nextToken)) {
1599                 conditions.notUidRange = tokens.nextToken();
1600             } else {
1601                 return appendNotSearchParams(nextToken, conditions);
1602             }
1603         } else if (token.startsWith("OR ")) {
1604             return appendOrSearchParams(token, conditions);
1605         } else if ("SUBJECT".equals(token)) {
1606             return session.contains("subject", tokens.nextToken());
1607         } else if ("BODY".equals(token)) {
1608             return session.contains("body", tokens.nextToken());
1609         } else if ("TEXT".equals(token)) {
1610             String value = tokens.nextToken();
1611             return session.or(session.contains("body", value),
1612                     session.contains("subject", value),
1613                     session.contains("from", value),
1614                     session.contains("to", value),
1615                     session.contains("cc", value));
1616         } else if ("KEYWORD".equals(token)) {
1617             return session.isEqualTo("keywords", session.convertFlagToKeyword(tokens.nextToken()));
1618         } else if ("UNKEYWORD".equals(token)) {
1619             return session.not(session.isEqualTo("keywords", session.convertFlagToKeyword(tokens.nextToken())));
1620         } else if ("FROM".equals(token)) {
1621             return session.contains("from", tokens.nextToken());
1622         } else if ("TO".equals(token)) {
1623             return session.contains("to", tokens.nextToken());
1624         } else if ("CC".equals(token)) {
1625             return session.contains("cc", tokens.nextToken());
1626         } else if ("LARGER".equals(token)) {
1627             return session.gte("messageSize", tokens.nextToken());
1628         } else if ("SMALLER".equals(token)) {
1629             return session.lt("messageSize", tokens.nextToken());
1630         } else if (token.startsWith("SENT") || "SINCE".equals(token) || "BEFORE".equals(token) || "ON".equals(token)) {
1631             return appendDateSearchParam(tokens, token);
1632         } else if ("SEEN".equals(token)) {
1633             return session.isTrue("read");
1634         } else if ("UNSEEN".equals(token) || "NEW".equals(token)) {
1635             return session.isFalse("read");
1636         } else if ("DRAFT".equals(token)) {
1637             conditions.draft = Boolean.TRUE;
1638         } else if ("UNDRAFT".equals(token)) {
1639             conditions.draft = Boolean.FALSE;
1640         } else if ("DELETED".equals(token)) {
1641             // conditions.deleted = Boolean.TRUE;
1642             return session.isEqualTo("deleted", "1");
1643         } else if ("UNDELETED".equals(token) || "NOT DELETED".equals(token)) {
1644             // conditions.deleted = Boolean.FALSE;
1645             return session.isNull("deleted");
1646         } else if ("FLAGGED".equals(token)) {
1647             conditions.flagged = Boolean.TRUE;
1648         } else if ("UNFLAGGED".equals(token)) {
1649             conditions.flagged = Boolean.FALSE;
1650         } else if ("ANSWERED".equals(token)) {
1651             conditions.answered = Boolean.TRUE;
1652         } else if ("UNANSWERED".equals(token)) {
1653             conditions.answered = Boolean.FALSE;
1654         } else if ("HEADER".equals(token)) {
1655             String headerName = tokens.nextToken().toLowerCase();
1656             String value = tokens.nextToken();
1657             if ("message-id".equals(headerName) && !value.startsWith("<")) {
1658                 value = '<' + value + '>';
1659             }
1660             return session.headerIsEqualTo(headerName, value);
1661         } else if ("UID".equals(token)) {
1662             String range = tokens.nextToken();
1663             // ignore 1:* noop filter
1664             if (!"1:*".equals(range)) {
1665                 conditions.uidRange = range;
1666             }
1667         } else //noinspection StatementWithEmptyBody
1668             if ("OLD".equals(token) || "RECENT".equals(token) || "ALL".equals(token)) {
1669                 // ignore
1670             } else if (token.indexOf(':') >= 0 || token.matches("\\d+") || token.indexOf(',') >= 0) {
1671                 // range search
1672                 conditions.indexRange = token;
1673             } else {
1674                 throw new DavMailException("EXCEPTION_INVALID_SEARCH_PARAMETERS", token);
1675             }
1676         // client side search token
1677         return null;
1678     }
1679 
1680     protected ExchangeSession.Condition appendDateSearchParam(ImapTokenizer tokens, String token) throws IOException {
1681         Date startDate;
1682         Date endDate;
1683         SimpleDateFormat parser = new SimpleDateFormat("dd-MMM-yyyy", Locale.ENGLISH);
1684         parser.setTimeZone(ExchangeSession.GMT_TIMEZONE);
1685         String dateToken = tokens.nextToken();
1686         try {
1687             startDate = parser.parse(dateToken);
1688             Calendar calendar = Calendar.getInstance();
1689             calendar.setTime(startDate);
1690             calendar.add(Calendar.DAY_OF_MONTH, 1);
1691             endDate = calendar.getTime();
1692         } catch (ParseException e) {
1693             throw new DavMailException("EXCEPTION_INVALID_SEARCH_PARAMETERS", dateToken);
1694         }
1695         String searchAttribute = "date";
1696 
1697         if (token.endsWith("ON")) {
1698             return session.and(session.gt(searchAttribute, session.formatSearchDate(startDate)),
1699                     session.lt(searchAttribute, session.formatSearchDate(endDate)));
1700         } else if (token.endsWith("BEFORE")) {
1701             return session.lt(searchAttribute, session.formatSearchDate(startDate));
1702         } else if (token.endsWith("SINCE")) {
1703             return session.gte(searchAttribute, session.formatSearchDate(startDate));
1704         } else {
1705             throw new DavMailException("EXCEPTION_INVALID_SEARCH_PARAMETERS", dateToken);
1706         }
1707     }
1708 
1709     protected boolean expunge(boolean silent) throws IOException {
1710         return expunge(silent, null);
1711     }
1712 
1713     protected boolean expunge(boolean silent, Set<Long> uidSet) throws IOException {
1714         boolean hasDeleted = false;
1715         if (currentFolder.messages != null) {
1716             int index = 1;
1717             for (ExchangeSession.Message message : currentFolder.messages) {
1718                 if (message.deleted && (uidSet == null || uidSet.contains(message.getImapUid()))) {
1719                     message.delete();
1720                     hasDeleted = true;
1721                     if (!silent) {
1722                         sendClient("* " + index + " EXPUNGE");
1723                     }
1724                 } else {
1725                     index++;
1726                 }
1727             }
1728         }
1729         return hasDeleted;
1730     }
1731 
1732     protected void updateFlags(ExchangeSession.Message message, String action, String flags) throws IOException {
1733         HashMap<String, String> properties = new HashMap<>();
1734         if ("-Flags".equalsIgnoreCase(action) || "-FLAGS.SILENT".equalsIgnoreCase(action)) {
1735             ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
1736             while (flagtokenizer.hasMoreTokens()) {
1737                 String flag = flagtokenizer.nextToken();
1738                 if ("\\Seen".equalsIgnoreCase(flag)) {
1739                     if (message.read) {
1740                         properties.put("read", "0");
1741                         message.read = false;
1742                     }
1743                 } else if ("\\Flagged".equalsIgnoreCase(flag)) {
1744                     if (message.flagged) {
1745                         properties.put("flagged", "0");
1746                         message.flagged = false;
1747                     }
1748                 } else if ("\\Deleted".equalsIgnoreCase(flag)) {
1749                     if (message.deleted) {
1750                         properties.put("deleted", null);
1751                         message.deleted = false;
1752                     }
1753                 } else if ("Junk".equalsIgnoreCase(flag)) {
1754                     if (message.junk) {
1755                         properties.put("junk", "0");
1756                         message.junk = false;
1757                     }
1758                 } else if ("$Forwarded".equalsIgnoreCase(flag)) {
1759                     if (message.forwarded) {
1760                         properties.put("forwarded", null);
1761                         message.forwarded = false;
1762                     }
1763                 } else if ("\\Answered".equalsIgnoreCase(flag)) {
1764                     if (message.answered) {
1765                         properties.put("answered", null);
1766                         message.answered = false;
1767                     }
1768                 } else //noinspection StatementWithEmptyBody
1769                     if ("\\Draft".equalsIgnoreCase(flag)) {
1770                         // ignore, draft is readonly after create
1771                     } else if (message.keywords != null) {
1772                         properties.put("keywords", message.removeFlag(flag));
1773                     }
1774             }
1775         } else if ("+Flags".equalsIgnoreCase(action) || "+FLAGS.SILENT".equalsIgnoreCase(action)) {
1776             ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
1777             while (flagtokenizer.hasMoreTokens()) {
1778                 String flag = flagtokenizer.nextToken();
1779                 if ("\\Seen".equalsIgnoreCase(flag)) {
1780                     if (!message.read) {
1781                         properties.put("read", "1");
1782                         message.read = true;
1783                     }
1784                 } else if ("\\Deleted".equalsIgnoreCase(flag)) {
1785                     if (!message.deleted) {
1786                         message.deleted = true;
1787                         properties.put("deleted", "1");
1788                     }
1789                 } else if ("\\Flagged".equalsIgnoreCase(flag)) {
1790                     if (!message.flagged) {
1791                         properties.put("flagged", "2");
1792                         message.flagged = true;
1793                     }
1794                 } else if ("\\Answered".equalsIgnoreCase(flag)) {
1795                     if (!message.answered) {
1796                         properties.put("answered", "102");
1797                         message.answered = true;
1798                     }
1799                 } else if ("$Forwarded".equalsIgnoreCase(flag)) {
1800                     if (!message.forwarded) {
1801                         properties.put("forwarded", "104");
1802                         message.forwarded = true;
1803                     }
1804                 } else if ("Junk".equalsIgnoreCase(flag)) {
1805                     if (!message.junk) {
1806                         properties.put("junk", "1");
1807                         message.junk = true;
1808                     }
1809                 } else //noinspection StatementWithEmptyBody
1810                     if ("\\Draft".equalsIgnoreCase(flag)) {
1811                         // ignore, draft is readonly after create
1812                     } else {
1813                         properties.put("keywords", message.addFlag(flag));
1814                     }
1815             }
1816         } else if ("FLAGS".equalsIgnoreCase(action) || "FLAGS.SILENT".equalsIgnoreCase(action)) {
1817             // flag list with default values
1818             boolean read = false;
1819             boolean deleted = false;
1820             boolean junk = false;
1821             boolean flagged = false;
1822             boolean answered = false;
1823             boolean forwarded = false;
1824             HashSet<String> keywords = null;
1825             // set flags from new flag list
1826             ImapTokenizer flagtokenizer = new ImapTokenizer(flags);
1827             while (flagtokenizer.hasMoreTokens()) {
1828                 String flag = flagtokenizer.nextToken();
1829                 if ("\\Seen".equalsIgnoreCase(flag)) {
1830                     read = true;
1831                 } else if ("\\Deleted".equalsIgnoreCase(flag)) {
1832                     deleted = true;
1833                 } else if ("\\Flagged".equalsIgnoreCase(flag)) {
1834                     flagged = true;
1835                 } else if ("\\Answered".equalsIgnoreCase(flag)) {
1836                     answered = true;
1837                 } else if ("$Forwarded".equalsIgnoreCase(flag)) {
1838                     forwarded = true;
1839                 } else if ("Junk".equalsIgnoreCase(flag)) {
1840                     junk = true;
1841                 } else //noinspection StatementWithEmptyBody
1842                     if ("\\Draft".equalsIgnoreCase(flag)) {
1843                         // ignore, draft is readonly after create
1844                     } else {
1845                         if (keywords == null) {
1846                             keywords = new HashSet<>();
1847                         }
1848                         keywords.add(flag);
1849                     }
1850             }
1851             if (keywords != null) {
1852                 properties.put("keywords", message.setFlags(keywords));
1853             }
1854             if (read != message.read) {
1855                 message.read = read;
1856                 if (message.read) {
1857                     properties.put("read", "1");
1858                 } else {
1859                     properties.put("read", "0");
1860                 }
1861             }
1862             if (deleted != message.deleted) {
1863                 message.deleted = deleted;
1864                 if (message.deleted) {
1865                     properties.put("deleted", "1");
1866                 } else {
1867                     properties.put("deleted", null);
1868                 }
1869             }
1870             if (flagged != message.flagged) {
1871                 message.flagged = flagged;
1872                 if (message.flagged) {
1873                     properties.put("flagged", "2");
1874                 } else {
1875                     properties.put("flagged", "0");
1876                 }
1877             }
1878             if (answered != message.answered) {
1879                 message.answered = answered;
1880                 if (message.answered) {
1881                     properties.put("answered", "102");
1882                 } else if (!forwarded) {
1883                     // remove property only if not forwarded
1884                     properties.put("answered", null);
1885                 }
1886             }
1887             if (forwarded != message.forwarded) {
1888                 message.forwarded = forwarded;
1889                 if (message.forwarded) {
1890                     properties.put("forwarded", "104");
1891                 } else if (!answered) {
1892                     // remove property only if not answered
1893                     properties.put("forwarded", null);
1894                 }
1895             }
1896             if (junk != message.junk) {
1897                 message.junk = junk;
1898                 if (message.junk) {
1899                     properties.put("junk", "1");
1900                 } else {
1901                     properties.put("junk", "0");
1902                 }
1903             }
1904         }
1905         if (!properties.isEmpty()) {
1906             session.updateMessage(message, properties);
1907             // message is no longer recent
1908             message.recent = false;
1909         }
1910     }
1911 
1912     /**
1913      * Decode IMAP credentials
1914      *
1915      * @param tokens tokens
1916      * @throws IOException on error
1917      */
1918     protected void parseCredentials(ImapTokenizer tokens) throws IOException {
1919         if (tokens.hasMoreTokens()) {
1920             userName = tokens.nextToken();
1921         } else {
1922             throw new DavMailException("EXCEPTION_INVALID_CREDENTIALS");
1923         }
1924 
1925         if (tokens.hasMoreTokens()) {
1926             password = tokens.nextToken();
1927         } else {
1928             throw new DavMailException("EXCEPTION_INVALID_CREDENTIALS");
1929         }
1930     }
1931 
1932     /**
1933      * Filter to output only headers, also count full size
1934      */
1935     private static final class PartOutputStream extends FilterOutputStream {
1936         protected enum State {
1937             START, CR, CRLF, CRLFCR, BODY
1938         }
1939 
1940         private State state = State.START;
1941         private int size;
1942         private int bufferSize;
1943         private final boolean writeHeaders;
1944         private final boolean writeBody;
1945         private final int startIndex;
1946         private final int maxSize;
1947 
1948         private PartOutputStream(OutputStream os, boolean writeHeaders, boolean writeBody,
1949                                  int startIndex, int maxSize) {
1950             super(os);
1951             this.writeHeaders = writeHeaders;
1952             this.writeBody = writeBody;
1953             this.startIndex = startIndex;
1954             this.maxSize = maxSize;
1955         }
1956 
1957         @Override
1958         public void write(int b) throws IOException {
1959             size++;
1960             if (((state != State.BODY && writeHeaders) || (state == State.BODY && writeBody)) &&
1961                     (size > startIndex) && (bufferSize < maxSize)
1962             ) {
1963                 super.write(b);
1964                 bufferSize++;
1965             }
1966             if (state == State.START) {
1967                 if (b == '\r') {
1968                     state = State.CR;
1969                 }
1970             } else if (state == State.CR) {
1971                 if (b == '\n') {
1972                     state = State.CRLF;
1973                 } else {
1974                     state = State.START;
1975                 }
1976             } else if (state == State.CRLF) {
1977                 if (b == '\r') {
1978                     state = State.CRLFCR;
1979                 } else {
1980                     state = State.START;
1981                 }
1982             } else if (state == State.CRLFCR) {
1983                 if (b == '\n') {
1984                     state = State.BODY;
1985                 } else {
1986                     state = State.START;
1987                 }
1988             }
1989         }
1990     }
1991 
1992     /**
1993      * Partial output stream, start at startIndex and write maxSize bytes.
1994      */
1995     private static final class PartialOutputStream extends FilterOutputStream {
1996         private int size;
1997         private int bufferSize;
1998         private final int startIndex;
1999         private final int maxSize;
2000 
2001         private PartialOutputStream(OutputStream os, int startIndex, int maxSize) {
2002             super(os);
2003             this.startIndex = startIndex;
2004             this.maxSize = maxSize;
2005         }
2006 
2007         @Override
2008         public void write(int b) throws IOException {
2009             size++;
2010             if ((size > startIndex) && (bufferSize < maxSize)) {
2011                 super.write(b);
2012                 bufferSize++;
2013             }
2014         }
2015     }
2016 
2017     protected abstract static class AbstractRangeIterator implements Iterator<ExchangeSession.Message> {
2018         ExchangeSession.MessageList messages;
2019         int currentIndex;
2020 
2021         protected int getCurrentIndex() {
2022             return currentIndex;
2023         }
2024     }
2025 
2026     protected static class UIDRangeIterator extends AbstractRangeIterator {
2027         final String[] ranges;
2028         int currentRangeIndex;
2029         long startUid;
2030         long endUid;
2031 
2032         protected UIDRangeIterator(ExchangeSession.MessageList messages, String value) {
2033             this.messages = messages;
2034             ranges = value.split(",");
2035         }
2036 
2037         protected long convertToLong(String value) {
2038             if ("*".equals(value)) {
2039                 return Long.MAX_VALUE;
2040             } else {
2041                 return Long.parseLong(value);
2042             }
2043         }
2044 
2045         protected void skipToNextRangeStartUid() {
2046             if (currentRangeIndex < ranges.length) {
2047                 String currentRange = ranges[currentRangeIndex++];
2048                 int colonIndex = currentRange.indexOf(':');
2049                 if (colonIndex > 0) {
2050                     startUid = convertToLong(currentRange.substring(0, colonIndex));
2051                     endUid = convertToLong(currentRange.substring(colonIndex + 1));
2052                     if (endUid < startUid) {
2053                         long swap = endUid;
2054                         endUid = startUid;
2055                         startUid = swap;
2056                     }
2057                 } else if ("*".equals(currentRange)) {
2058                     startUid = endUid = messages.get(messages.size() - 1).getImapUid();
2059                 } else {
2060                     startUid = endUid = convertToLong(currentRange);
2061                 }
2062                 while (currentIndex < messages.size() && messages.get(currentIndex).getImapUid() < startUid) {
2063                     currentIndex++;
2064                 }
2065             } else {
2066                 currentIndex = messages.size();
2067             }
2068         }
2069 
2070         protected boolean hasNextInRange() {
2071             return hasNextIndex() && messages.get(currentIndex).getImapUid() <= endUid;
2072         }
2073 
2074         protected boolean hasNextIndex() {
2075             return currentIndex < messages.size();
2076         }
2077 
2078         protected boolean hasNextRange() {
2079             return currentRangeIndex < ranges.length;
2080         }
2081 
2082         public boolean hasNext() {
2083             boolean hasNextInRange = hasNextInRange();
2084             // if has next range and current index after current range end, reset index
2085             if (hasNextRange() && !hasNextInRange) {
2086                 currentIndex = 0;
2087             }
2088             while (hasNextIndex() && !hasNextInRange) {
2089                 skipToNextRangeStartUid();
2090                 hasNextInRange = hasNextInRange();
2091             }
2092             return hasNextIndex();
2093         }
2094 
2095         public ExchangeSession.Message next() {
2096             ExchangeSession.Message message = messages.get(currentIndex++);
2097             long uid = message.getImapUid();
2098             if (uid < startUid || uid > endUid) {
2099                 throw new NoSuchElementException("Message uid " + uid + " not in range " + startUid + ':' + endUid);
2100             }
2101             return message;
2102         }
2103 
2104         public void remove() {
2105             throw new UnsupportedOperationException();
2106         }
2107     }
2108 
2109     protected static class RangeIterator extends AbstractRangeIterator {
2110         final String[] ranges;
2111         int currentRangeIndex;
2112         long startUid;
2113         long endUid;
2114 
2115         protected RangeIterator(ExchangeSession.MessageList messages, String value) {
2116             this.messages = messages;
2117             ranges = value.split(",");
2118         }
2119 
2120         protected long convertToLong(String value) {
2121             if ("*".equals(value)) {
2122                 return Long.MAX_VALUE;
2123             } else {
2124                 return Long.parseLong(value);
2125             }
2126         }
2127 
2128         protected void skipToNextRangeStart() {
2129             if (currentRangeIndex < ranges.length) {
2130                 String currentRange = ranges[currentRangeIndex++];
2131                 int colonIndex = currentRange.indexOf(':');
2132                 if (colonIndex > 0) {
2133                     startUid = convertToLong(currentRange.substring(0, colonIndex));
2134                     endUid = convertToLong(currentRange.substring(colonIndex + 1));
2135                     if (endUid < startUid) {
2136                         long swap = endUid;
2137                         endUid = startUid;
2138                         startUid = swap;
2139                     }
2140                 } else if ("*".equals(currentRange)) {
2141                     startUid = endUid = messages.size();
2142                 } else {
2143                     startUid = endUid = convertToLong(currentRange);
2144                 }
2145                 while (currentIndex < messages.size() && (currentIndex + 1) < startUid) {
2146                     currentIndex++;
2147                 }
2148             } else {
2149                 currentIndex = messages.size();
2150             }
2151         }
2152 
2153         protected boolean hasNextInRange() {
2154             return hasNextIndex() && currentIndex < endUid;
2155         }
2156 
2157         protected boolean hasNextIndex() {
2158             return currentIndex < messages.size();
2159         }
2160 
2161         protected boolean hasNextRange() {
2162             return currentRangeIndex < ranges.length;
2163         }
2164 
2165         public boolean hasNext() {
2166             boolean hasNextInRange = hasNextInRange();
2167             // if has next range and current index after current range end, reset index
2168             if (hasNextRange() && !hasNextInRange) {
2169                 currentIndex = 0;
2170             }
2171             while (hasNextIndex() && !hasNextInRange) {
2172                 skipToNextRangeStart();
2173                 hasNextInRange = hasNextInRange();
2174             }
2175             return hasNextIndex();
2176         }
2177 
2178         public ExchangeSession.Message next() {
2179             if (currentIndex >= messages.size()) {
2180                 throw new NoSuchElementException();
2181             }
2182             return messages.get(currentIndex++);
2183         }
2184 
2185         public void remove() {
2186             throw new UnsupportedOperationException();
2187         }
2188     }
2189 
2190     static protected class ImapTokenizer {
2191         char[] value;
2192         int startIndex;
2193         ArrayDeque<Character> quotes = new ArrayDeque<>();
2194 
2195         ImapTokenizer(String value) {
2196             this.value = value.toCharArray();
2197         }
2198 
2199         public String nextToken() {
2200             // Get next token without removing quotes ", {} or ()
2201             String token = nextQuotedToken();
2202             // note: literal strings not handled here.
2203             if( !token.isEmpty() && '"' == token.charAt(0) ) {
2204                 // token is quoted string.
2205                 try {
2206                     token = StringUtil.parseQuotedImapString(token);
2207                 } catch (ParseException e) {
2208                     LOGGER.warn("Invalid quoted token: "+token);
2209                     token = StringUtil.removeQuotes(token);
2210                 }
2211             } else {
2212                 // use the general method previously also used;
2213                 // for example unquotes a list. I guess naming could be made better in the future.
2214                 token = StringUtil.removeQuotes(token);
2215             }
2216             return token;
2217         }
2218 
2219         protected boolean isQuote(char character) {
2220             return character == '"' || character == '(' || character == ')' ||
2221                     character == '[' || character == ']' || character == '\\';
2222         }
2223 
2224         public boolean hasMoreTokens() {
2225             return startIndex < value.length;
2226         }
2227 
2228         public String nextQuotedToken() {
2229             int currentIndex = startIndex;
2230             while (currentIndex < value.length) {
2231                 char currentChar = value[currentIndex];
2232                 if (currentChar == ' ' && quotes.isEmpty()) {
2233                     break;
2234                 } else if (!quotes.isEmpty() && quotes.peek() == '\\') {
2235                     // just skip
2236                     quotes.pop();
2237                 } else if (isQuote(currentChar)) {
2238                     if (quotes.isEmpty()) {
2239                         quotes.push(currentChar);
2240                     } else {
2241                         char currentQuote = quotes.peek();
2242                         if (currentChar == '\\') {
2243                             quotes.push(currentChar);
2244                         } else if (currentQuote == '"' && currentChar == '"' ||
2245                                 currentQuote == '(' && currentChar == ')' ||
2246                                 currentQuote == '[' && currentChar == ']'
2247                         ) {
2248                             // end quote
2249                             quotes.pop();
2250                         } else {
2251                             quotes.push(currentChar);
2252                         }
2253                     }
2254                 }
2255                 currentIndex++;
2256             }
2257             String result = new String(value, startIndex, currentIndex - startIndex);
2258             startIndex = currentIndex + 1;
2259             return result;
2260         }
2261     }
2262 
2263 }