Recently, I got to, as many many years ago, dive in Windows API, to implement a rather simple task. I needed to restrict an FMX-based application, namely its Windows-flavour target, to be single-instance, so any attempt to launch yet anther copy of its process would (try to) bring up the first running one.

First, I got to say, that this solution is immature, kind of quick drop-in, and later it may grow into something more universal, regarding MAX OS platform as well. That's said, let's create two helper classes right in main project file code.

Unfold, to see implementation:

class EsSingleInstanceGuard
{
public:
  EsSingleInstanceGuard(const UnicodeString& name) :
  m_ok(false),
  m_duplicate(false),
  m_name(name)
#if ES_OS == ES_OS_WINDOWS
  ,
  m_mxGlobal(NULL),
  m_mx(NULL),
  m_msgId(0)
#endif
  {
#if ES_OS == ES_OS_WINDOWS
    SECURITY_DESCRIPTOR securityDesc;
    SECURITY_ATTRIBUTES securityAttr;

    memset( &securityDesc, sizeof(securityDesc), 0 );
    memset( &securityAttr, sizeof(securityAttr), 0 );

    ::InitializeSecurityDescriptor(
      &securityDesc,
      SECURITY_DESCRIPTOR_REVISION
    );

    ::SetSecurityDescriptorDacl(
      &securityDesc,
      TRUE,
      nullptr,
      FALSE
    );

    securityAttr.nLength = sizeof(securityAttr);
    securityAttr.lpSecurityDescriptor = &securityDesc;
    securityAttr.bInheritHandle = FALSE;

    m_mx = ::CreateMutex(
      &securityAttr,
      FALSE,
      m_name.c_str()
    );

    if( NULL == m_mx )
      return;

    if( ERROR_ALREADY_EXISTS == GetLastError() )
      m_duplicate = true;

    UnicodeString mxNameGlobal = "Global\\" + m_name;
    m_mxGlobal = ::CreateMutex(
      &securityAttr,
      FALSE,
      mxNameGlobal.c_str()
    );

    if( NULL == m_mxGlobal )
      return;

    if( ERROR_ALREADY_EXISTS == GetLastError() )
      m_duplicate = true;

    const UnicodeString& msgGuard = "SINGLE_APP_GUARD_"+m_name;
    m_msgId = ::RegisterWindowMessage(
      msgGuard.c_str()
    );
#else

#endif

    m_ok = true;
  }

  ~EsSingleInstanceGuard()
  {
#if ES_OS == ES_OS_WINDOWS
    if( m_mx )
    {
      ::CloseHandle(m_mx);
      m_mx = NULL;
    }

    if( m_mxGlobal )
    {
      ::CloseHandle(m_mxGlobal);
      m_mxGlobal = NULL;
    }
#endif
  }

  void notifyInstance()
  {
    if( m_msgId )
      ::SendMessage(
        HWND_BROADCAST,
        m_msgId,
        0,
        0
      );
  }

  __property bool ok = {read=m_ok};
  __property bool duplicate = {read=m_duplicate};
#if ES_OS == ES_OS_WINDOWS
  __property UINT messageId = {read=m_msgId};
#endif

protected:
  bool m_ok;
  bool m_duplicate;
  UnicodeString m_name;
#if ES_OS == ES_OS_WINDOWS
  HANDLE m_mxGlobal;
  HANDLE m_mx;
  UINT m_msgId;
#endif

private:
  EsSingleInstanceGuard() = delete;
  EsSingleInstanceGuard(const EsSingleInstanceGuard&) = delete;
  EsSingleInstanceGuard& operator=(const EsSingleInstanceGuard&) = delete;
};
//---------------------------------------------------------------------------

#if ES_OS == ES_OS_WINDOWS
# include <FMX.Platform.Win.hpp>
class EsSingleInstanceGuardMsgHook
{
public:
  EsSingleInstanceGuardMsgHook(UINT guardMsgId) :
  m_hwnd( Fmx::Platform::Win::ApplicationHWND() ),
  m_oldProc(0),
  m_guardMsgId(guardMsgId),
  m_installed(false)
  {
    s_inst = this;
    m_oldProc = ::SetWindowLongPtr(
      m_hwnd,
      GWLP_WNDPROC,
      reinterpret_cast<LONG_PTR>(&HookWndProc)
    );

    m_installed = 0 != m_oldProc;
  }

  ~EsSingleInstanceGuardMsgHook()
  {
    if( m_installed )
      ::SetWindowLongPtr(
        m_hwnd,
        GWLP_WNDPROC,
        m_oldProc
      );

    m_installed = false;
    m_hwnd = nullptr;
    m_oldProc = 0;
    s_inst = nullptr;
  }

protected:
  static LRESULT CALLBACK HookWndProc(
    HWND   hwnd,
    UINT   uMsg,
    WPARAM wParam,
    LPARAM lParam
  )
  {
    if( s_inst &&
        uMsg == s_inst->m_guardMsgId &&
        hwnd == s_inst->m_hwnd
    )
    {
      if( Application->MainForm )
      {
        Application->MainForm->Show();
      }

      return 0;
    }

    return ::CallWindowProc(
      reinterpret_cast<WNDPROC>(s_inst->m_oldProc),
      hwnd,
      uMsg,
      wParam,
      lParam
    );
  }

protected:
  HWND m_hwnd;
  LONG_PTR m_oldProc;
  UINT m_guardMsgId;
  bool m_installed;
  static EsSingleInstanceGuardMsgHook* s_inst;

private:
  EsSingleInstanceGuardMsgHook() = delete;
  EsSingleInstanceGuardMsgHook(const EsSingleInstanceGuardMsgHook&) = delete;
  EsSingleInstanceGuardMsgHook& operator=(const EsSingleInstanceGuardMsgHook&) = delete;
};

EsSingleInstanceGuardMsgHook* EsSingleInstanceGuardMsgHook::s_inst = nullptr;

#endif
//---------------------------------------------------------------------------

'At do we 'ave 'ere? :)

First, EsSingleInstanceGuard is a singleton, which should be created once an execution is reached main FMX entry point. Internally, it creates|opens an existing Named OS Mutexes. Named OS Windows Mutex is very useful here, because it exists at OS scope, not local to process, so different processes may request access to the same named mutex object.

Wht two named mutexes? Because one exists as User-local namespace, and another one, with Global prefix, apparently, at a global scope, which is shared among all User sessions.

Mutexes base name may naturally be a generated GUID string. In code snippet below, I use one GUID for x86 process, and another for 64 bit one.

In addition, Guard's class constructor registers the named Windows window message, and stores its ID in m_guardMsgId member. Wofur? That's the way we later notify the first instance to show itself on top of other windows. The good thing for named messages is that all processes which register it, will get the same message id. The guard's notifyInstance code does simply that, namely, sends a broadcast Windows window message, which is being dispatched to all process's main windows. The notifyInstance is performed only if guard detects it's a duplicate instance. After notification is sent, a duplicate immediately exits, before any other initializing actions are done.

The only step is left, is to allow our application to handle that broadcast message.

Static singleton initialization at the start of FMX entry point+multiple instance checking code:

extern "C" int FMXmain()
{
  // Initialize named mutexes for single-instance execution guard and detection
  static EsSingleInstanceGuard s_guard(
#ifdef _WIN64
    "ED2C0D81-319F-4D98-862F-95A3477318B6"
#else
    "6F425A4C-D7A0-4F7D-8E1A-8565634F0D2A"
#endif
  );

  if( s_guard.ok && s_guard.duplicate )
  {
    s_guard.notifyInstance();

    return 0;
  }

To allow our application to handle Guard broadcast message, whe got to implement so-called, Instance Subclassing for Application's internal HWND object. The second singleton helper class, EsSingleInstanceGuardMsgHook does exactly that.

EsSingleInstanceGuardMsgHook singleton usage:

#if ES_OS == ES_OS_WINDOWS
    EsSingleInstanceGuardMsgHook guardHook(
      s_guard.messageId
    );
#endif

    Application->Run();

It overrides, in a Windows way, the Application HWND's Windows Procedure, at the first step it checks and handles guard message, if detected, otherwise, it passes message handling to the "base class" procedure.

Yet one more positive side-effect of using named mutexes, is its application in Innosetup Installer Engine, which may use these mutex names to detect if application is running at the moment installer is launched, and prompt user to close an application, or exit installer.