Commandline: Do some checks related to custom-charset options if user specifies them

Fixes https://github.com/hashcat/hashcat/issues/1040

docs/changes.txt

@@ -51,6 +51,7 @@
 - Building: Add SHARED variable to Makefile to choose if hashcat is build as static or shared binary (using libhashcat.so/hashcat.dll)
 - Building: Removed the use of RPATH on linker level
 - Building: Replaced linking of CRT_glob.o with the use of int _dowildcard
+- Commandline: Do some checks related to custom-charset options if user specifies them
 - Events: Improved the maximum event message handling. event_log () will now also internally make sure that the message is properly terminated
 - Files: Do several file and folder checks on startup rather than when they are actually used to avoid related error after eventual intense operations
 - Helper: Added functions to check existence, type, read- and write-permissions and rewrite sources to use them instead of stat()

src/user_options.c

@@ -774,6 +774,34 @@ int user_options_sanity (hashcat_ctx_t *hashcat_ctx)
     }
   }
 
+  // custom charset checks
+
+  if ((user_options->custom_charset_1 != NULL)
+   || (user_options->custom_charset_2 != NULL)
+   || (user_options->custom_charset_3 != NULL)
+   || (user_options->custom_charset_4 != NULL))
+  {
+    if (user_options->attack_mode == ATTACK_MODE_STRAIGHT)
+    {
+      event_log_error (hashcat_ctx, "Custom-charsets not supported in attack-mode 0");
+
+      return -1;
+    }
+    else if (user_options->attack_mode == ATTACK_MODE_COMBI)
+    {
+      event_log_error (hashcat_ctx, "Custom-charsets not supported in attack-mode 1");
+
+      return -1;
+    }
+
+    if (user_options->hc_argc < 2)
+    {
+      event_log_error (hashcat_ctx, "You need to specify a mask if you specify a custom-charset");
+
+      return -1;
+    }
+  }
+
   // argc / argv checks
 
   bool show_error = true;