001/*
002 * Copyright 2008-2018 Ping Identity Corporation
003 * All Rights Reserved.
004 */
005/*
006 * Copyright (C) 2008-2018 Ping Identity Corporation
007 *
008 * This program is free software; you can redistribute it and/or modify
009 * it under the terms of the GNU General Public License (GPLv2 only)
010 * or the terms of the GNU Lesser General Public License (LGPLv2.1 only)
011 * as published by the Free Software Foundation.
012 *
013 * This program is distributed in the hope that it will be useful,
014 * but WITHOUT ANY WARRANTY; without even the implied warranty of
015 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
016 * GNU General Public License for more details.
017 *
018 * You should have received a copy of the GNU General Public License
019 * along with this program; if not, see <http://www.gnu.org/licenses>.
020 */
021package com.unboundid.ldap.sdk.examples;
022
023
024
025import java.io.IOException;
026import java.io.OutputStream;
027import java.io.Serializable;
028import java.text.ParseException;
029import java.util.ArrayList;
030import java.util.LinkedHashMap;
031import java.util.LinkedHashSet;
032import java.util.List;
033import java.util.Random;
034import java.util.concurrent.CyclicBarrier;
035import java.util.concurrent.atomic.AtomicBoolean;
036import java.util.concurrent.atomic.AtomicLong;
037
038import com.unboundid.ldap.sdk.Control;
039import com.unboundid.ldap.sdk.LDAPConnection;
040import com.unboundid.ldap.sdk.LDAPConnectionOptions;
041import com.unboundid.ldap.sdk.LDAPException;
042import com.unboundid.ldap.sdk.ResultCode;
043import com.unboundid.ldap.sdk.Version;
044import com.unboundid.ldap.sdk.controls.AssertionRequestControl;
045import com.unboundid.ldap.sdk.controls.PermissiveModifyRequestControl;
046import com.unboundid.ldap.sdk.controls.PostReadRequestControl;
047import com.unboundid.ldap.sdk.controls.PreReadRequestControl;
048import com.unboundid.util.ColumnFormatter;
049import com.unboundid.util.FixedRateBarrier;
050import com.unboundid.util.FormattableColumn;
051import com.unboundid.util.HorizontalAlignment;
052import com.unboundid.util.LDAPCommandLineTool;
053import com.unboundid.util.ObjectPair;
054import com.unboundid.util.OutputFormat;
055import com.unboundid.util.RateAdjustor;
056import com.unboundid.util.ResultCodeCounter;
057import com.unboundid.util.ThreadSafety;
058import com.unboundid.util.ThreadSafetyLevel;
059import com.unboundid.util.ValuePattern;
060import com.unboundid.util.WakeableSleeper;
061import com.unboundid.util.args.ArgumentException;
062import com.unboundid.util.args.ArgumentParser;
063import com.unboundid.util.args.BooleanArgument;
064import com.unboundid.util.args.ControlArgument;
065import com.unboundid.util.args.FileArgument;
066import com.unboundid.util.args.FilterArgument;
067import com.unboundid.util.args.IntegerArgument;
068import com.unboundid.util.args.StringArgument;
069
070import static com.unboundid.util.Debug.*;
071import static com.unboundid.util.StaticUtils.*;
072
073
074
075/**
076 * This class provides a tool that can be used to perform repeated modifications
077 * in an LDAP directory server using multiple threads.  It can help provide an
078 * estimate of the modify performance that a directory server is able to
079 * achieve.  The target entry DN may be a value pattern as described in the
080 * {@link ValuePattern} class.  This makes it possible to modify a range of
081 * entries rather than repeatedly updating the same entry.
082 * <BR><BR>
083 * Some of the APIs demonstrated by this example include:
084 * <UL>
085 *   <LI>Argument Parsing (from the {@code com.unboundid.util.args}
086 *       package)</LI>
087 *   <LI>LDAP Command-Line Tool (from the {@code com.unboundid.util}
088 *       package)</LI>
089 *   <LI>LDAP Communication (from the {@code com.unboundid.ldap.sdk}
090 *       package)</LI>
091 *   <LI>Value Patterns (from the {@code com.unboundid.util} package)</LI>
092 * </UL>
093 * <BR><BR>
094 * All of the necessary information is provided using command line arguments.
095 * Supported arguments include those allowed by the {@link LDAPCommandLineTool}
096 * class, as well as the following additional arguments:
097 * <UL>
098 *   <LI>"-b {entryDN}" or "--targetDN {baseDN}" -- specifies the DN of the
099 *       entry to be modified.  This must be provided.  It may be a simple DN,
100 *       or it may be a value pattern to express a range of entry DNs.</LI>
101 *   <LI>"-A {name}" or "--attribute {name}" -- specifies the name of the
102 *       attribute to modify.  Multiple attributes may be modified by providing
103 *       multiple instances of this argument.  At least one attribute must be
104 *       provided.</LI>
105 *   <LI>"-l {num}" or "--valueLength {num}" -- specifies the length in bytes to
106 *       use for the values of the target attributes.  If this is not provided,
107 *       then a default length of 10 bytes will be used.</LI>
108 *   <LI>"-C {chars}" or "--characterSet {chars}" -- specifies the set of
109 *       characters that will be used to generate the values to use for the
110 *       target attributes.  It should only include ASCII characters.  Values
111 *       will be generated from randomly-selected characters from this set.  If
112 *       this is not provided, then a default set of lowercase alphabetic
113 *       characters will be used.</LI>
114 *   <LI>"-t {num}" or "--numThreads {num}" -- specifies the number of
115 *       concurrent threads to use when performing the modifications.  If this
116 *       is not provided, then a default of one thread will be used.</LI>
117 *   <LI>"-i {sec}" or "--intervalDuration {sec}" -- specifies the length of
118 *       time in seconds between lines out output.  If this is not provided,
119 *       then a default interval duration of five seconds will be used.</LI>
120 *   <LI>"-I {num}" or "--numIntervals {num}" -- specifies the maximum number of
121 *       intervals for which to run.  If this is not provided, then it will
122 *       run forever.</LI>
123 *   <LI>"--iterationsBeforeReconnect {num}" -- specifies the number of modify
124 *       iterations that should be performed on a connection before that
125 *       connection is closed and replaced with a newly-established (and
126 *       authenticated, if appropriate) connection.</LI>
127 *   <LI>"-r {modifies-per-second}" or "--ratePerSecond {modifies-per-second}"
128 *       -- specifies the target number of modifies to perform per second.  It
129 *       is still necessary to specify a sufficient number of threads for
130 *       achieving this rate.  If this option is not provided, then the tool
131 *       will run at the maximum rate for the specified number of threads.</LI>
132 *   <LI>"--variableRateData {path}" -- specifies the path to a file containing
133 *       information needed to allow the tool to vary the target rate over time.
134 *       If this option is not provided, then the tool will either use a fixed
135 *       target rate as specified by the "--ratePerSecond" argument, or it will
136 *       run at the maximum rate.</LI>
137 *   <LI>"--generateSampleRateFile {path}" -- specifies the path to a file to
138 *       which sample data will be written illustrating and describing the
139 *       format of the file expected to be used in conjunction with the
140 *       "--variableRateData" argument.</LI>
141 *   <LI>"--warmUpIntervals {num}" -- specifies the number of intervals to
142 *       complete before beginning overall statistics collection.</LI>
143 *   <LI>"--timestampFormat {format}" -- specifies the format to use for
144 *       timestamps included before each output line.  The format may be one of
145 *       "none" (for no timestamps), "with-date" (to include both the date and
146 *       the time), or "without-date" (to include only time time).</LI>
147 *   <LI>"-Y {authzID}" or "--proxyAs {authzID}" -- Use the proxied
148 *       authorization v2 control to request that the operation be processed
149 *       using an alternate authorization identity.  In this case, the bind DN
150 *       should be that of a user that has permission to use this control.  The
151 *       authorization identity may be a value pattern.</LI>
152 *   <LI>"--suppressErrorResultCodes" -- Indicates that information about the
153 *       result codes for failed operations should not be displayed.</LI>
154 *   <LI>"-c" or "--csv" -- Generate output in CSV format rather than a
155 *       display-friendly format.</LI>
156 * </UL>
157 */
158@ThreadSafety(level=ThreadSafetyLevel.NOT_THREADSAFE)
159public final class ModRate
160       extends LDAPCommandLineTool
161       implements Serializable
162{
163  /**
164   * The serial version UID for this serializable class.
165   */
166  private static final long serialVersionUID = 2709717414202815822L;
167
168
169
170  // Indicates whether a request has been made to stop running.
171  private final AtomicBoolean stopRequested;
172
173  // The argument used to indicate whether to generate output in CSV format.
174  private BooleanArgument csvFormat;
175
176  // Indicates that the tool should use the increment modification type instead
177  // of replace.
178  private BooleanArgument increment;
179
180  // Indicates that modify requests should include the permissive modify request
181  // control.
182  private BooleanArgument permissiveModify;
183
184  // The argument used to indicate whether to suppress information about error
185  // result codes.
186  private BooleanArgument suppressErrorsArgument;
187
188  // The argument used to indicate that a generic control should be included in
189  // the request.
190  private ControlArgument control;
191
192  // The argument used to specify a variable rate file.
193  private FileArgument sampleRateFile;
194
195  // The argument used to specify a variable rate file.
196  private FileArgument variableRateData;
197
198  // Indicates that modify requests should include the assertion request control
199  // with the specified filter.
200  private FilterArgument assertionFilter;
201
202  // The argument used to specify the collection interval.
203  private IntegerArgument collectionInterval;
204
205  // The increment amount to use when performing an increment instead of a
206  // replace.
207  private IntegerArgument incrementAmount;
208
209  // The argument used to specify the number of modify iterations on a
210  // connection before it is closed and re-established.
211  private IntegerArgument iterationsBeforeReconnect;
212
213  // The argument used to specify the number of intervals.
214  private IntegerArgument numIntervals;
215
216  // The argument used to specify the number of threads.
217  private IntegerArgument numThreads;
218
219  // The argument used to specify the seed to use for the random number
220  // generator.
221  private IntegerArgument randomSeed;
222
223  // The target rate of modifies per second.
224  private IntegerArgument ratePerSecond;
225
226  // The number of values to include in the replace modification.
227  private IntegerArgument valueCount;
228
229  // The argument used to specify the length of the values to generate.
230  private IntegerArgument valueLength;
231
232  // The number of warm-up intervals to perform.
233  private IntegerArgument warmUpIntervals;
234
235  // The argument used to specify the name of the attribute to modify.
236  private StringArgument attribute;
237
238  // The argument used to specify the set of characters to use when generating
239  // values.
240  private StringArgument characterSet;
241
242  // The argument used to specify the DNs of the entries to modify.
243  private StringArgument entryDN;
244
245  // Indicates that modify requests should include the post-read request control
246  // to request the specified attribute.
247  private StringArgument postReadAttribute;
248
249  // Indicates that modify requests should include the pre-read request control
250  // to request the specified attribute.
251  private StringArgument preReadAttribute;
252
253  // The argument used to specify the proxied authorization identity.
254  private StringArgument proxyAs;
255
256  // The argument used to specify the timestamp format.
257  private StringArgument timestampFormat;
258
259  // The thread currently being used to run the searchrate tool.
260  private volatile Thread runningThread;
261
262  // A wakeable sleeper that will be used to sleep between reporting intervals.
263  private final WakeableSleeper sleeper;
264
265
266
267  /**
268   * Parse the provided command line arguments and make the appropriate set of
269   * changes.
270   *
271   * @param  args  The command line arguments provided to this program.
272   */
273  public static void main(final String[] args)
274  {
275    final ResultCode resultCode = main(args, System.out, System.err);
276    if (resultCode != ResultCode.SUCCESS)
277    {
278      System.exit(resultCode.intValue());
279    }
280  }
281
282
283
284  /**
285   * Parse the provided command line arguments and make the appropriate set of
286   * changes.
287   *
288   * @param  args       The command line arguments provided to this program.
289   * @param  outStream  The output stream to which standard out should be
290   *                    written.  It may be {@code null} if output should be
291   *                    suppressed.
292   * @param  errStream  The output stream to which standard error should be
293   *                    written.  It may be {@code null} if error messages
294   *                    should be suppressed.
295   *
296   * @return  A result code indicating whether the processing was successful.
297   */
298  public static ResultCode main(final String[] args,
299                                final OutputStream outStream,
300                                final OutputStream errStream)
301  {
302    final ModRate modRate = new ModRate(outStream, errStream);
303    return modRate.runTool(args);
304  }
305
306
307
308  /**
309   * Creates a new instance of this tool.
310   *
311   * @param  outStream  The output stream to which standard out should be
312   *                    written.  It may be {@code null} if output should be
313   *                    suppressed.
314   * @param  errStream  The output stream to which standard error should be
315   *                    written.  It may be {@code null} if error messages
316   *                    should be suppressed.
317   */
318  public ModRate(final OutputStream outStream, final OutputStream errStream)
319  {
320    super(outStream, errStream);
321
322    stopRequested = new AtomicBoolean(false);
323    sleeper = new WakeableSleeper();
324  }
325
326
327
328  /**
329   * Retrieves the name for this tool.
330   *
331   * @return  The name for this tool.
332   */
333  @Override()
334  public String getToolName()
335  {
336    return "modrate";
337  }
338
339
340
341  /**
342   * Retrieves the description for this tool.
343   *
344   * @return  The description for this tool.
345   */
346  @Override()
347  public String getToolDescription()
348  {
349    return "Perform repeated modifications against " +
350           "an LDAP directory server.";
351  }
352
353
354
355  /**
356   * Retrieves the version string for this tool.
357   *
358   * @return  The version string for this tool.
359   */
360  @Override()
361  public String getToolVersion()
362  {
363    return Version.NUMERIC_VERSION_STRING;
364  }
365
366
367
368  /**
369   * Indicates whether this tool should provide support for an interactive mode,
370   * in which the tool offers a mode in which the arguments can be provided in
371   * a text-driven menu rather than requiring them to be given on the command
372   * line.  If interactive mode is supported, it may be invoked using the
373   * "--interactive" argument.  Alternately, if interactive mode is supported
374   * and {@link #defaultsToInteractiveMode()} returns {@code true}, then
375   * interactive mode may be invoked by simply launching the tool without any
376   * arguments.
377   *
378   * @return  {@code true} if this tool supports interactive mode, or
379   *          {@code false} if not.
380   */
381  @Override()
382  public boolean supportsInteractiveMode()
383  {
384    return true;
385  }
386
387
388
389  /**
390   * Indicates whether this tool defaults to launching in interactive mode if
391   * the tool is invoked without any command-line arguments.  This will only be
392   * used if {@link #supportsInteractiveMode()} returns {@code true}.
393   *
394   * @return  {@code true} if this tool defaults to using interactive mode if
395   *          launched without any command-line arguments, or {@code false} if
396   *          not.
397   */
398  @Override()
399  public boolean defaultsToInteractiveMode()
400  {
401    return true;
402  }
403
404
405
406  /**
407   * Indicates whether this tool should provide arguments for redirecting output
408   * to a file.  If this method returns {@code true}, then the tool will offer
409   * an "--outputFile" argument that will specify the path to a file to which
410   * all standard output and standard error content will be written, and it will
411   * also offer a "--teeToStandardOut" argument that can only be used if the
412   * "--outputFile" argument is present and will cause all output to be written
413   * to both the specified output file and to standard output.
414   *
415   * @return  {@code true} if this tool should provide arguments for redirecting
416   *          output to a file, or {@code false} if not.
417   */
418  @Override()
419  protected boolean supportsOutputFile()
420  {
421    return true;
422  }
423
424
425
426  /**
427   * Indicates whether this tool should default to interactively prompting for
428   * the bind password if a password is required but no argument was provided
429   * to indicate how to get the password.
430   *
431   * @return  {@code true} if this tool should default to interactively
432   *          prompting for the bind password, or {@code false} if not.
433   */
434  @Override()
435  protected boolean defaultToPromptForBindPassword()
436  {
437    return true;
438  }
439
440
441
442  /**
443   * Indicates whether this tool supports the use of a properties file for
444   * specifying default values for arguments that aren't specified on the
445   * command line.
446   *
447   * @return  {@code true} if this tool supports the use of a properties file
448   *          for specifying default values for arguments that aren't specified
449   *          on the command line, or {@code false} if not.
450   */
451  @Override()
452  public boolean supportsPropertiesFile()
453  {
454    return true;
455  }
456
457
458
459  /**
460   * Indicates whether the LDAP-specific arguments should include alternate
461   * versions of all long identifiers that consist of multiple words so that
462   * they are available in both camelCase and dash-separated versions.
463   *
464   * @return  {@code true} if this tool should provide multiple versions of
465   *          long identifiers for LDAP-specific arguments, or {@code false} if
466   *          not.
467   */
468  @Override()
469  protected boolean includeAlternateLongIdentifiers()
470  {
471    return true;
472  }
473
474
475
476  /**
477   * {@inheritDoc}
478   */
479  @Override()
480  protected boolean logToolInvocationByDefault()
481  {
482    return true;
483  }
484
485
486
487  /**
488   * Adds the arguments used by this program that aren't already provided by the
489   * generic {@code LDAPCommandLineTool} framework.
490   *
491   * @param  parser  The argument parser to which the arguments should be added.
492   *
493   * @throws  ArgumentException  If a problem occurs while adding the arguments.
494   */
495  @Override()
496  public void addNonLDAPArguments(final ArgumentParser parser)
497         throws ArgumentException
498  {
499    String description = "The DN of the entry to modify.  It may be a simple " +
500         "DN or a value pattern to specify a range of DN (e.g., " +
501         "\"uid=user.[1-1000],ou=People,dc=example,dc=com\").  See " +
502         ValuePattern.PUBLIC_JAVADOC_URL + " for complete details about the " +
503         "value pattern syntax.  This must be provided.";
504    entryDN = new StringArgument('b', "entryDN", true, 1, "{dn}", description);
505    entryDN.setArgumentGroupName("Modification Arguments");
506    entryDN.addLongIdentifier("entry-dn", true);
507    parser.addArgument(entryDN);
508
509
510    description = "The name of the attribute to modify.  Multiple attributes " +
511                  "may be specified by providing this argument multiple " +
512                  "times.  At least one attribute must be specified.";
513    attribute = new StringArgument('A', "attribute", true, 0, "{name}",
514                                   description);
515    attribute.setArgumentGroupName("Modification Arguments");
516    parser.addArgument(attribute);
517
518
519    description = "The length in bytes to use when generating values for the " +
520                  "replace modifications.  If this is not provided, then a " +
521                  "default length of ten bytes will be used.";
522    valueLength = new IntegerArgument('l', "valueLength", true, 1, "{num}",
523                                      description, 1, Integer.MAX_VALUE, 10);
524    valueLength.setArgumentGroupName("Modification Arguments");
525    valueLength.addLongIdentifier("value-length", true);
526    parser.addArgument(valueLength);
527
528
529    description = "The number of values to include in replace " +
530                  "modifications.  If this is not provided, then a default " +
531                  "of one value will be used.";
532    valueCount = new IntegerArgument(null, "valueCount", false, 1, "{num}",
533                                     description, 0, Integer.MAX_VALUE, 1);
534    valueCount.setArgumentGroupName("Modification Arguments");
535    valueCount.addLongIdentifier("value-count", true);
536    parser.addArgument(valueCount);
537
538
539    description = "Indicates that the tool should use the increment " +
540                  "modification type rather than the replace modification " +
541                  "type.";
542    increment = new BooleanArgument(null, "increment", 1, description);
543    increment.setArgumentGroupName("Modification Arguments");
544    parser.addArgument(increment);
545
546
547    description = "The amount by which to increment values when using the " +
548                  "increment modification type.  The amount may be negative " +
549                  "if values should be decremented rather than incremented.  " +
550                  "If this is not provided, then a default increment amount " +
551                  "of one will be used.";
552    incrementAmount = new IntegerArgument(null, "incrementAmount", false, 1,
553                                          null, description, Integer.MIN_VALUE,
554                                          Integer.MAX_VALUE, 1);
555    incrementAmount.setArgumentGroupName("Modification Arguments");
556    incrementAmount.addLongIdentifier("increment-amount", true);
557    parser.addArgument(incrementAmount);
558
559
560    description = "The set of characters to use to generate the values for " +
561                  "the modifications.  It should only include ASCII " +
562                  "characters.  If this is not provided, then a default set " +
563                  "of lowercase alphabetic characters will be used.";
564    characterSet = new StringArgument('C', "characterSet", true, 1, "{chars}",
565                                      description,
566                                      "abcdefghijklmnopqrstuvwxyz");
567    characterSet.setArgumentGroupName("Modification Arguments");
568    characterSet.addLongIdentifier("character-set", true);
569    parser.addArgument(characterSet);
570
571
572    description = "Indicates that modify requests should include the " +
573                  "assertion request control with the specified filter.";
574    assertionFilter = new FilterArgument(null, "assertionFilter", false, 1,
575                                         "{filter}", description);
576    assertionFilter.setArgumentGroupName("Request Control Arguments");
577    assertionFilter.addLongIdentifier("assertion-filter", true);
578    parser.addArgument(assertionFilter);
579
580
581    description = "Indicates that modify requests should include the " +
582                  "permissive modify request control.";
583    permissiveModify = new BooleanArgument(null, "permissiveModify", 1,
584                                           description);
585    permissiveModify.setArgumentGroupName("Request Control Arguments");
586    permissiveModify.addLongIdentifier("permissive-modify", true);
587    parser.addArgument(permissiveModify);
588
589
590    description = "Indicates that modify requests should include the " +
591                  "pre-read request control with the specified requested " +
592                  "attribute.  This argument may be provided multiple times " +
593                  "to indicate that multiple requested attributes should be " +
594                  "included in the pre-read request control.";
595    preReadAttribute = new StringArgument(null, "preReadAttribute", false, 0,
596                                          "{attribute}", description);
597    preReadAttribute.setArgumentGroupName("Request Control Arguments");
598    preReadAttribute.addLongIdentifier("pre-read-attribute", true);
599    parser.addArgument(preReadAttribute);
600
601
602    description = "Indicates that modify requests should include the " +
603                  "post-read request control with the specified requested " +
604                  "attribute.  This argument may be provided multiple times " +
605                  "to indicate that multiple requested attributes should be " +
606                  "included in the post-read request control.";
607    postReadAttribute = new StringArgument(null, "postReadAttribute", false, 0,
608                                           "{attribute}", description);
609    postReadAttribute.setArgumentGroupName("Request Control Arguments");
610    postReadAttribute.addLongIdentifier("post-read-attribute", true);
611    parser.addArgument(postReadAttribute);
612
613
614    description = "Indicates that the proxied authorization control (as " +
615                  "defined in RFC 4370) should be used to request that " +
616                  "operations be processed using an alternate authorization " +
617                  "identity.  This may be a simple authorization ID or it " +
618                  "may be a value pattern to specify a range of " +
619                  "identities.  See " + ValuePattern.PUBLIC_JAVADOC_URL +
620                  " for complete details about the value pattern syntax.";
621    proxyAs = new StringArgument('Y', "proxyAs", false, 1, "{authzID}",
622                                 description);
623    proxyAs.setArgumentGroupName("Request Control Arguments");
624    proxyAs.addLongIdentifier("proxy-as", true);
625    parser.addArgument(proxyAs);
626
627
628    description = "Indicates that modify requests should include the " +
629                  "specified request control.  This may be provided multiple " +
630                  "times to include multiple request controls.";
631    control = new ControlArgument('J', "control", false, 0, null, description);
632    control.setArgumentGroupName("Request Control Arguments");
633    parser.addArgument(control);
634
635
636    description = "The number of threads to use to perform the " +
637                  "modifications.  If this is not provided, a single thread " +
638                  "will be used.";
639    numThreads = new IntegerArgument('t', "numThreads", true, 1, "{num}",
640                                     description, 1, Integer.MAX_VALUE, 1);
641    numThreads.setArgumentGroupName("Rate Management Arguments");
642    numThreads.addLongIdentifier("num-threads", true);
643    parser.addArgument(numThreads);
644
645
646    description = "The length of time in seconds between output lines.  If " +
647                  "this is not provided, then a default interval of five " +
648                  "seconds will be used.";
649    collectionInterval = new IntegerArgument('i', "intervalDuration", true, 1,
650                                             "{num}", description, 1,
651                                             Integer.MAX_VALUE, 5);
652    collectionInterval.setArgumentGroupName("Rate Management Arguments");
653    collectionInterval.addLongIdentifier("interval-duration", true);
654    parser.addArgument(collectionInterval);
655
656
657    description = "The maximum number of intervals for which to run.  If " +
658                  "this is not provided, then the tool will run until it is " +
659                  "interrupted.";
660    numIntervals = new IntegerArgument('I', "numIntervals", true, 1, "{num}",
661                                       description, 1, Integer.MAX_VALUE,
662                                       Integer.MAX_VALUE);
663    numIntervals.setArgumentGroupName("Rate Management Arguments");
664    numIntervals.addLongIdentifier("num-intervals", true);
665    parser.addArgument(numIntervals);
666
667    description = "The number of modify iterations that should be processed " +
668                  "on a connection before that connection is closed and " +
669                  "replaced with a newly-established (and authenticated, if " +
670                  "appropriate) connection.  If this is not provided, then " +
671                  "connections will not be periodically closed and " +
672                  "re-established.";
673    iterationsBeforeReconnect = new IntegerArgument(null,
674         "iterationsBeforeReconnect", false, 1, "{num}", description, 0);
675    iterationsBeforeReconnect.setArgumentGroupName("Rate Management Arguments");
676    iterationsBeforeReconnect.addLongIdentifier("iterations-before-reconnect",
677         true);
678    parser.addArgument(iterationsBeforeReconnect);
679
680    description = "The target number of modifies to perform per second.  It " +
681                  "is still necessary to specify a sufficient number of " +
682                  "threads for achieving this rate.  If neither this option " +
683                  "nor --variableRateData is provided, then the tool will " +
684                  "run at the maximum rate for the specified number of " +
685                  "threads.";
686    ratePerSecond = new IntegerArgument('r', "ratePerSecond", false, 1,
687                                        "{modifies-per-second}", description,
688                                        1, Integer.MAX_VALUE);
689    ratePerSecond.setArgumentGroupName("Rate Management Arguments");
690    ratePerSecond.addLongIdentifier("rate-per-second", true);
691    parser.addArgument(ratePerSecond);
692
693    final String variableRateDataArgName = "variableRateData";
694    final String generateSampleRateFileArgName = "generateSampleRateFile";
695    description = RateAdjustor.getVariableRateDataArgumentDescription(
696         generateSampleRateFileArgName);
697    variableRateData = new FileArgument(null, variableRateDataArgName, false, 1,
698                                        "{path}", description, true, true, true,
699                                        false);
700    variableRateData.setArgumentGroupName("Rate Management Arguments");
701    variableRateData.addLongIdentifier("variable-rate-data", true);
702    parser.addArgument(variableRateData);
703
704    description = RateAdjustor.getGenerateSampleVariableRateFileDescription(
705         variableRateDataArgName);
706    sampleRateFile = new FileArgument(null, generateSampleRateFileArgName,
707                                      false, 1, "{path}", description, false,
708                                      true, true, false);
709    sampleRateFile.setArgumentGroupName("Rate Management Arguments");
710    sampleRateFile.addLongIdentifier("generate-sample-rate-file", true);
711    sampleRateFile.setUsageArgument(true);
712    parser.addArgument(sampleRateFile);
713    parser.addExclusiveArgumentSet(variableRateData, sampleRateFile);
714
715    description = "The number of intervals to complete before beginning " +
716                  "overall statistics collection.  Specifying a nonzero " +
717                  "number of warm-up intervals gives the client and server " +
718                  "a chance to warm up without skewing performance results.";
719    warmUpIntervals = new IntegerArgument(null, "warmUpIntervals", true, 1,
720         "{num}", description, 0, Integer.MAX_VALUE, 0);
721    warmUpIntervals.setArgumentGroupName("Rate Management Arguments");
722    warmUpIntervals.addLongIdentifier("warm-up-intervals", true);
723    parser.addArgument(warmUpIntervals);
724
725    description = "Indicates the format to use for timestamps included in " +
726                  "the output.  A value of 'none' indicates that no " +
727                  "timestamps should be included.  A value of 'with-date' " +
728                  "indicates that both the date and the time should be " +
729                  "included.  A value of 'without-date' indicates that only " +
730                  "the time should be included.";
731    final LinkedHashSet<String> allowedFormats = new LinkedHashSet<String>(3);
732    allowedFormats.add("none");
733    allowedFormats.add("with-date");
734    allowedFormats.add("without-date");
735    timestampFormat = new StringArgument(null, "timestampFormat", true, 1,
736         "{format}", description, allowedFormats, "none");
737    timestampFormat.addLongIdentifier("timestamp-format", true);
738    parser.addArgument(timestampFormat);
739
740    description = "Indicates that information about the result codes for " +
741                  "failed operations should not be displayed.";
742    suppressErrorsArgument = new BooleanArgument(null,
743         "suppressErrorResultCodes", 1, description);
744    suppressErrorsArgument.addLongIdentifier("suppress-error-result-codes",
745         true);
746    parser.addArgument(suppressErrorsArgument);
747
748    description = "Generate output in CSV format rather than a " +
749                  "display-friendly format";
750    csvFormat = new BooleanArgument('c', "csv", 1, description);
751    parser.addArgument(csvFormat);
752
753    description = "Specifies the seed to use for the random number generator.";
754    randomSeed = new IntegerArgument('R', "randomSeed", false, 1, "{value}",
755         description);
756    randomSeed.addLongIdentifier("random-seed", true);
757    parser.addArgument(randomSeed);
758
759
760    // The incrementAmount argument can only be used if the increment argument
761    // is provided.
762    parser.addDependentArgumentSet(incrementAmount, increment);
763
764
765    // Neither the valueLength nor valueCount arguments can be used if the
766    // increment argument is provided.
767    parser.addExclusiveArgumentSet(increment, valueLength);
768    parser.addExclusiveArgumentSet(increment, valueCount);
769  }
770
771
772
773  /**
774   * Indicates whether this tool supports creating connections to multiple
775   * servers.  If it is to support multiple servers, then the "--hostname" and
776   * "--port" arguments will be allowed to be provided multiple times, and
777   * will be required to be provided the same number of times.  The same type of
778   * communication security and bind credentials will be used for all servers.
779   *
780   * @return  {@code true} if this tool supports creating connections to
781   *          multiple servers, or {@code false} if not.
782   */
783  @Override()
784  protected boolean supportsMultipleServers()
785  {
786    return true;
787  }
788
789
790
791  /**
792   * Retrieves the connection options that should be used for connections
793   * created for use with this tool.
794   *
795   * @return  The connection options that should be used for connections created
796   *          for use with this tool.
797   */
798  @Override()
799  public LDAPConnectionOptions getConnectionOptions()
800  {
801    final LDAPConnectionOptions options = new LDAPConnectionOptions();
802    options.setUseSynchronousMode(true);
803    return options;
804  }
805
806
807
808  /**
809   * Performs the actual processing for this tool.  In this case, it gets a
810   * connection to the directory server and uses it to perform the requested
811   * modifications.
812   *
813   * @return  The result code for the processing that was performed.
814   */
815  @Override()
816  public ResultCode doToolProcessing()
817  {
818    runningThread = Thread.currentThread();
819
820    try
821    {
822      return doToolProcessingInternal();
823    }
824    finally
825    {
826      runningThread = null;
827    }
828
829  }
830
831
832  /**
833   * Performs the actual processing for this tool.  In this case, it gets a
834   * connection to the directory server and uses it to perform the requested
835   * modifications.
836   *
837   * @return  The result code for the processing that was performed.
838   */
839  private ResultCode doToolProcessingInternal()
840  {
841    // If the sample rate file argument was specified, then generate the sample
842    // variable rate data file and return.
843    if (sampleRateFile.isPresent())
844    {
845      try
846      {
847        RateAdjustor.writeSampleVariableRateFile(sampleRateFile.getValue());
848        return ResultCode.SUCCESS;
849      }
850      catch (final Exception e)
851      {
852        debugException(e);
853        err("An error occurred while trying to write sample variable data " +
854             "rate file '", sampleRateFile.getValue().getAbsolutePath(),
855             "':  ", getExceptionMessage(e));
856        return ResultCode.LOCAL_ERROR;
857      }
858    }
859
860
861    // Determine the random seed to use.
862    final Long seed;
863    if (randomSeed.isPresent())
864    {
865      seed = Long.valueOf(randomSeed.getValue());
866    }
867    else
868    {
869      seed = null;
870    }
871
872    // Create the value patterns for the target entry DN and proxied
873    // authorization identities.
874    final ValuePattern dnPattern;
875    try
876    {
877      dnPattern = new ValuePattern(entryDN.getValue(), seed);
878    }
879    catch (final ParseException pe)
880    {
881      debugException(pe);
882      err("Unable to parse the entry DN value pattern:  ", pe.getMessage());
883      return ResultCode.PARAM_ERROR;
884    }
885
886    final ValuePattern authzIDPattern;
887    if (proxyAs.isPresent())
888    {
889      try
890      {
891        authzIDPattern = new ValuePattern(proxyAs.getValue(), seed);
892      }
893      catch (final ParseException pe)
894      {
895        debugException(pe);
896        err("Unable to parse the proxied authorization pattern:  ",
897            pe.getMessage());
898        return ResultCode.PARAM_ERROR;
899      }
900    }
901    else
902    {
903      authzIDPattern = null;
904    }
905
906
907    // Get the set of controls to include in modify requests.
908    final ArrayList<Control> controlList = new ArrayList<Control>(5);
909    if (assertionFilter.isPresent())
910    {
911      controlList.add(new AssertionRequestControl(assertionFilter.getValue()));
912    }
913
914    if (permissiveModify.isPresent())
915    {
916      controlList.add(new PermissiveModifyRequestControl());
917    }
918
919    if (preReadAttribute.isPresent())
920    {
921      final List<String> attrList = preReadAttribute.getValues();
922      final String[] attrArray = new String[attrList.size()];
923      attrList.toArray(attrArray);
924      controlList.add(new PreReadRequestControl(attrArray));
925    }
926
927    if (postReadAttribute.isPresent())
928    {
929      final List<String> attrList = postReadAttribute.getValues();
930      final String[] attrArray = new String[attrList.size()];
931      attrList.toArray(attrArray);
932      controlList.add(new PostReadRequestControl(attrArray));
933    }
934
935    if (control.isPresent())
936    {
937      controlList.addAll(control.getValues());
938    }
939
940    final Control[] controlArray = new Control[controlList.size()];
941    controlList.toArray(controlArray);
942
943
944    // Get the names of the attributes to modify.
945    final String[] attrs = new String[attribute.getValues().size()];
946    attribute.getValues().toArray(attrs);
947
948
949    // Get the character set as a byte array.
950    final byte[] charSet = getBytes(characterSet.getValue());
951
952
953    // If the --ratePerSecond option was specified, then limit the rate
954    // accordingly.
955    FixedRateBarrier fixedRateBarrier = null;
956    if (ratePerSecond.isPresent() || variableRateData.isPresent())
957    {
958      // We might not have a rate per second if --variableRateData is specified.
959      // The rate typically doesn't matter except when we have warm-up
960      // intervals.  In this case, we'll run at the max rate.
961      final int intervalSeconds = collectionInterval.getValue();
962      final int ratePerInterval =
963           (ratePerSecond.getValue() == null)
964           ? Integer.MAX_VALUE
965           : ratePerSecond.getValue() * intervalSeconds;
966      fixedRateBarrier =
967           new FixedRateBarrier(1000L * intervalSeconds, ratePerInterval);
968    }
969
970
971    // If --variableRateData was specified, then initialize a RateAdjustor.
972    RateAdjustor rateAdjustor = null;
973    if (variableRateData.isPresent())
974    {
975      try
976      {
977        rateAdjustor = RateAdjustor.newInstance(fixedRateBarrier,
978             ratePerSecond.getValue(), variableRateData.getValue());
979      }
980      catch (final IOException e)
981      {
982        debugException(e);
983        err("Initializing the variable rates failed: " + e.getMessage());
984        return ResultCode.PARAM_ERROR;
985      }
986      catch (final IllegalArgumentException e)
987      {
988        debugException(e);
989        err("Initializing the variable rates failed: " + e.getMessage());
990        return ResultCode.PARAM_ERROR;
991      }
992    }
993
994
995    // Determine whether to include timestamps in the output and if so what
996    // format should be used for them.
997    final boolean includeTimestamp;
998    final String timeFormat;
999    if (timestampFormat.getValue().equalsIgnoreCase("with-date"))
1000    {
1001      includeTimestamp = true;
1002      timeFormat       = "dd/MM/yyyy HH:mm:ss";
1003    }
1004    else if (timestampFormat.getValue().equalsIgnoreCase("without-date"))
1005    {
1006      includeTimestamp = true;
1007      timeFormat       = "HH:mm:ss";
1008    }
1009    else
1010    {
1011      includeTimestamp = false;
1012      timeFormat       = null;
1013    }
1014
1015
1016    // Determine whether any warm-up intervals should be run.
1017    final long totalIntervals;
1018    final boolean warmUp;
1019    int remainingWarmUpIntervals = warmUpIntervals.getValue();
1020    if (remainingWarmUpIntervals > 0)
1021    {
1022      warmUp = true;
1023      totalIntervals = 0L + numIntervals.getValue() + remainingWarmUpIntervals;
1024    }
1025    else
1026    {
1027      warmUp = true;
1028      totalIntervals = 0L + numIntervals.getValue();
1029    }
1030
1031
1032    // Create the table that will be used to format the output.
1033    final OutputFormat outputFormat;
1034    if (csvFormat.isPresent())
1035    {
1036      outputFormat = OutputFormat.CSV;
1037    }
1038    else
1039    {
1040      outputFormat = OutputFormat.COLUMNS;
1041    }
1042
1043    final ColumnFormatter formatter = new ColumnFormatter(includeTimestamp,
1044         timeFormat, outputFormat, " ",
1045         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1046                  "Mods/Sec"),
1047         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1048                  "Avg Dur ms"),
1049         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Recent",
1050                  "Errors/Sec"),
1051         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1052                  "Mods/Sec"),
1053         new FormattableColumn(12, HorizontalAlignment.RIGHT, "Overall",
1054                  "Avg Dur ms"));
1055
1056
1057    // Create values to use for statistics collection.
1058    final AtomicLong        modCounter   = new AtomicLong(0L);
1059    final AtomicLong        errorCounter = new AtomicLong(0L);
1060    final AtomicLong        modDurations = new AtomicLong(0L);
1061    final ResultCodeCounter rcCounter    = new ResultCodeCounter();
1062
1063
1064    // Determine the length of each interval in milliseconds.
1065    final long intervalMillis = 1000L * collectionInterval.getValue();
1066
1067
1068    // Create a random number generator to use for seeding the per-thread
1069    // generators.
1070    final Random random = new Random();
1071
1072
1073    // Create the threads to use for the modifications.
1074    final CyclicBarrier barrier = new CyclicBarrier(numThreads.getValue() + 1);
1075    final ModRateThread[] threads = new ModRateThread[numThreads.getValue()];
1076    for (int i=0; i < threads.length; i++)
1077    {
1078      final LDAPConnection connection;
1079      try
1080      {
1081        connection = getConnection();
1082      }
1083      catch (final LDAPException le)
1084      {
1085        debugException(le);
1086        err("Unable to connect to the directory server:  ",
1087            getExceptionMessage(le));
1088        return le.getResultCode();
1089      }
1090
1091      threads[i] = new ModRateThread(this, i, connection, dnPattern, attrs,
1092           charSet, valueLength.getValue(), valueCount.getValue(),
1093           increment.isPresent(), incrementAmount.getValue(), controlArray,
1094           authzIDPattern, random.nextLong(),
1095           iterationsBeforeReconnect.getValue(), barrier, modCounter,
1096           modDurations, errorCounter, rcCounter, fixedRateBarrier);
1097      threads[i].start();
1098    }
1099
1100
1101    // Display the table header.
1102    for (final String headerLine : formatter.getHeaderLines(true))
1103    {
1104      out(headerLine);
1105    }
1106
1107
1108    // Start the RateAdjustor before the threads so that the initial value is
1109    // in place before any load is generated unless we're doing a warm-up in
1110    // which case, we'll start it after the warm-up is complete.
1111    if ((rateAdjustor != null) && (remainingWarmUpIntervals <= 0))
1112    {
1113      rateAdjustor.start();
1114    }
1115
1116
1117    // Indicate that the threads can start running.
1118    try
1119    {
1120      barrier.await();
1121    }
1122    catch (final Exception e)
1123    {
1124      debugException(e);
1125    }
1126
1127    long overallStartTime = System.nanoTime();
1128    long nextIntervalStartTime = System.currentTimeMillis() + intervalMillis;
1129
1130
1131    boolean setOverallStartTime = false;
1132    long    lastDuration        = 0L;
1133    long    lastNumErrors       = 0L;
1134    long    lastNumMods         = 0L;
1135    long    lastEndTime         = System.nanoTime();
1136    for (long i=0; i < totalIntervals; i++)
1137    {
1138      if (rateAdjustor != null)
1139      {
1140        if (! rateAdjustor.isAlive())
1141        {
1142          out("All of the rates in " + variableRateData.getValue().getName() +
1143              " have been completed.");
1144          break;
1145        }
1146      }
1147
1148      final long startTimeMillis = System.currentTimeMillis();
1149      final long sleepTimeMillis = nextIntervalStartTime - startTimeMillis;
1150      nextIntervalStartTime += intervalMillis;
1151      if (sleepTimeMillis > 0)
1152      {
1153        sleeper.sleep(sleepTimeMillis);
1154      }
1155
1156      if (stopRequested.get())
1157      {
1158        break;
1159      }
1160
1161      final long endTime          = System.nanoTime();
1162      final long intervalDuration = endTime - lastEndTime;
1163
1164      final long numMods;
1165      final long numErrors;
1166      final long totalDuration;
1167      if (warmUp && (remainingWarmUpIntervals > 0))
1168      {
1169        numMods       = modCounter.getAndSet(0L);
1170        numErrors     = errorCounter.getAndSet(0L);
1171        totalDuration = modDurations.getAndSet(0L);
1172      }
1173      else
1174      {
1175        numMods       = modCounter.get();
1176        numErrors     = errorCounter.get();
1177        totalDuration = modDurations.get();
1178      }
1179
1180      final long recentNumMods = numMods - lastNumMods;
1181      final long recentNumErrors = numErrors - lastNumErrors;
1182      final long recentDuration = totalDuration - lastDuration;
1183
1184      final double numSeconds = intervalDuration / 1000000000.0d;
1185      final double recentModRate = recentNumMods / numSeconds;
1186      final double recentErrorRate  = recentNumErrors / numSeconds;
1187
1188      final double recentAvgDuration;
1189      if (recentNumMods > 0L)
1190      {
1191        recentAvgDuration = 1.0d * recentDuration / recentNumMods / 1000000;
1192      }
1193      else
1194      {
1195        recentAvgDuration = 0.0d;
1196      }
1197
1198      if (warmUp && (remainingWarmUpIntervals > 0))
1199      {
1200        out(formatter.formatRow(recentModRate, recentAvgDuration,
1201             recentErrorRate, "warming up", "warming up"));
1202
1203        remainingWarmUpIntervals--;
1204        if (remainingWarmUpIntervals == 0)
1205        {
1206          out("Warm-up completed.  Beginning overall statistics collection.");
1207          setOverallStartTime = true;
1208          if (rateAdjustor != null)
1209          {
1210            rateAdjustor.start();
1211          }
1212        }
1213      }
1214      else
1215      {
1216        if (setOverallStartTime)
1217        {
1218          overallStartTime    = lastEndTime;
1219          setOverallStartTime = false;
1220        }
1221
1222        final double numOverallSeconds =
1223             (endTime - overallStartTime) / 1000000000.0d;
1224        final double overallAuthRate = numMods / numOverallSeconds;
1225
1226        final double overallAvgDuration;
1227        if (numMods > 0L)
1228        {
1229          overallAvgDuration = 1.0d * totalDuration / numMods / 1000000;
1230        }
1231        else
1232        {
1233          overallAvgDuration = 0.0d;
1234        }
1235
1236        out(formatter.formatRow(recentModRate, recentAvgDuration,
1237             recentErrorRate, overallAuthRate, overallAvgDuration));
1238
1239        lastNumMods     = numMods;
1240        lastNumErrors   = numErrors;
1241        lastDuration    = totalDuration;
1242      }
1243
1244      final List<ObjectPair<ResultCode,Long>> rcCounts =
1245           rcCounter.getCounts(true);
1246      if ((! suppressErrorsArgument.isPresent()) && (! rcCounts.isEmpty()))
1247      {
1248        err("\tError Results:");
1249        for (final ObjectPair<ResultCode,Long> p : rcCounts)
1250        {
1251          err("\t", p.getFirst().getName(), ":  ", p.getSecond());
1252        }
1253      }
1254
1255      lastEndTime = endTime;
1256    }
1257
1258    // Shut down the RateAdjustor if we have one.
1259    if (rateAdjustor != null)
1260    {
1261      rateAdjustor.shutDown();
1262    }
1263
1264    // Stop all of the threads.
1265    ResultCode resultCode = ResultCode.SUCCESS;
1266    for (final ModRateThread t : threads)
1267    {
1268      final ResultCode r = t.stopRunning();
1269      if (resultCode == ResultCode.SUCCESS)
1270      {
1271        resultCode = r;
1272      }
1273    }
1274
1275    return resultCode;
1276  }
1277
1278
1279
1280  /**
1281   * Requests that this tool stop running.  This method will attempt to wait
1282   * for all threads to complete before returning control to the caller.
1283   */
1284  public void stopRunning()
1285  {
1286    stopRequested.set(true);
1287    sleeper.wakeup();
1288
1289    final Thread t = runningThread;
1290    if (t != null)
1291    {
1292      try
1293      {
1294        t.join();
1295      }
1296      catch (final Exception e)
1297      {
1298        debugException(e);
1299
1300        if (e instanceof InterruptedException)
1301        {
1302          Thread.currentThread().interrupt();
1303        }
1304      }
1305    }
1306  }
1307
1308
1309
1310  /**
1311   * {@inheritDoc}
1312   */
1313  @Override()
1314  public LinkedHashMap<String[],String> getExampleUsages()
1315  {
1316    final LinkedHashMap<String[],String> examples =
1317         new LinkedHashMap<String[],String>(2);
1318
1319    String[] args =
1320    {
1321      "--hostname", "server.example.com",
1322      "--port", "389",
1323      "--bindDN", "uid=admin,dc=example,dc=com",
1324      "--bindPassword", "password",
1325      "--entryDN", "uid=user.[1-1000000],ou=People,dc=example,dc=com",
1326      "--attribute", "description",
1327      "--valueLength", "12",
1328      "--numThreads", "10"
1329    };
1330    String description =
1331         "Test modify performance by randomly selecting entries across a set " +
1332         "of one million users located below 'ou=People,dc=example,dc=com' " +
1333         "with ten concurrent threads and replacing the values for the " +
1334         "description attribute with a string of 12 randomly-selected " +
1335         "lowercase alphabetic characters.";
1336    examples.put(args, description);
1337
1338    args = new String[]
1339    {
1340      "--generateSampleRateFile", "variable-rate-data.txt"
1341    };
1342    description =
1343         "Generate a sample variable rate definition file that may be used " +
1344         "in conjunction with the --variableRateData argument.  The sample " +
1345         "file will include comments that describe the format for data to be " +
1346         "included in this file.";
1347    examples.put(args, description);
1348
1349    return examples;
1350  }
1351}