Solution: Pick the CORRECT definition of extfn_cancel from the Help.

This Help topic http://dcx.sybase.com/index.html#1201/en/dbprogramming/pg-extfun-cancel.html

shows this (apparently correct) code...

extern "C" void SQL_CALLBACK extfn_cancel( void *cancel_handle )
{
    *(short *)cancel_handle = 1;
}

whereas this Help topic http://dcx.sybase.com/index.html#1201/en/dbprogramming/pg-extfun-uecar.html

shows this (apparently incorrect) code...

extern "C" __declspec( dllexport )
void extfn_cancel( void *cancel_handle )
{
    *(short *)cancel_handle = 1;
}

extern "C" __declspec( dllexport ) 
void mystring( an_extfn_api *api, void *arg_handle )
{
.
.
.
    short  canceled = 0;

extapi->set_cancel( arg_handle, &canceled );
.
.
.
    if( canceled )

There are two things wrong with the second code snippet, not just the different (and apparently wrong) definition of extfn_cancel.

The OTHER wrong thing, which I noticed but disregarded (silly me), was the reference to "extapi->" in the usage sample doesn't match the argument definition "an_extfn_api *api".

In other words, the Help code was (apparently) never tested.

Here's the funny thing: In my original code I posted the HTML link to the correct Help topic, but pasted the code from the WRONG topic! :)

For the record, here's the working CPP code, DEF file, SQL test and the output from the SELECT which was run after the CALL was stopped:

#include "windows.h"
#include "wininet.h"
#pragma comment(lib, "Wininet")
#include "extfnapi.h"

//----------------------------------------------------------------------------------------------------------
// extfn_use_new_api
//----------------------------------------------------------------------------------------------------------

extern "C" a_sql_uint32 SQL_CALLBACK extfn_use_new_api ( void )
{
   // SQL Anywhere external function call interface: extfn_use_new_api method
   //    http://dcx.sybase.com/index.html#1201/en/dbprogramming/pg-extfun-newapi.html

return( EXTFN_API_VERSION );
}

//----------------------------------------------------------------------------------------------------------
// extfn_cancel
//----------------------------------------------------------------------------------------------------------

extern "C" void SQL_CALLBACK extfn_cancel( void *cancel_handle )
{
   // SQL Anywhere external function call cancel: extfn_cancel method
   //    http://dcx.sybase.com/index.html#1201/en/dbprogramming/pg-extfun-cancel.html

*(short *)cancel_handle = 1;
}

//----------------------------------------------------------------------------------------------------------
// sqla_test_cancel
//----------------------------------------------------------------------------------------------------------

__declspec ( dllexport ) void FAR __stdcall sqla_test_cancel ( an_extfn_api *api, void *arg_handle ) 
{

an_extfn_value       api_loop_counter;
   unsigned long long   loop_counter;
   short                canceled;

canceled = 0;
   api -> set_cancel ( arg_handle, &canceled );

loop_counter = 0;

while ( canceled == 0 && loop_counter <= 1000000000000000 ) {
      loop_counter = loop_counter + 1;
      api -> set_cancel ( arg_handle, &canceled );
      if ( canceled == 0 ) {
         api_loop_counter.type = DT_UNSBIGINT;
         api_loop_counter.data = &loop_counter;
         api -> set_value ( arg_handle, 1, &api_loop_counter, FALSE );
      }
      api -> set_cancel ( arg_handle, &canceled );
   }
}

EXPORTS sqla_test_cancel
EXPORTS extfn_cancel
EXPORTS extfn_use_new_api

CREATE PROCEDURE sqla_test_cancel ( 
   OUT @loop_counter   UNSIGNED BIGINT )
   EXTERNAL NAME 'sqla_test_cancel@C:\\projects\\C\\sqla_test_cancel\\Release\\sqla_test_cancel.dll';

IF VAREXISTS ( '@loop_counter' ) = 1 THEN
   DROP VARIABLE @loop_counter;
END IF;

CREATE VARIABLE @loop_counter UNSIGNED BIGINT;

-- Click on SQL - Stop when after this CALL runs for a while...

CALL sqla_test_cancel ( @loop_counter );

-- Then run this SELECT to see how far the DLL loop got...

SELECT @loop_counter;

@loop_counter
78459977