You are not setting up the header record correctly. You need to use SetKeyValue on fields that make up the primary key.

See below revisions. Look for lines sandwiched between lines starting and ending with five # symbols.

Dim Answer As String
Dim MyNote As String
'Place your text here
MyNote = "Import AP invoice into Sage?"
'Display MessageBox
Answer = MsgBox(MyNote, vbQuestion + vbYesNo, "Orders")
If Answer = vbYes Then
	Me.Status = "Processing A/P Invoices"
	'DoCmd.OpenQuery "17 Update GL_Account Key"
	' Variables used for return values from ProvideX methods or as Parameters
	' that will not have their value changed.  No need to initialize.
	' Numeric return values
	Dim retVal, TaskID As Integer
	'Get the path to mas90\home which is required by the Init() method
	Set oReg = GetObject("winmgmts:\\.\root\default:StdRegProv")
	oReg.GetExpandedStringValue &H80000001, "Software\ODBC\ODBC.INI\MAS_SYSTEM", "Directory", PathRoot
	'pathhome = PathRoot & "\Home"
	pathhome = "\\vapp03\Sage\Sage 100 Premium 2018\MAS90\Home"
	Set oReg = Nothing
	'MsgBox "path " & pathhome
	'End If  '*** Test
	'End Sub
	'Private Sub test_Click()
	Set oScript = CreateObject("ProvideX.Script")
	'oScript.Init("C:\Program Files\Sage Software\MAS 90\Version4\MAS90\Home")
	oScript.Init (pathhome)
	Set oSS = oScript.NewObject("SY_SESSION")
	'The Logon method returns 1 if MAS90 Security is not enabled or if
	'MAS90 has been configured to use Windows Security
	retVal = oSS.nLogon()
	'If the Logon method fails then you must use the SetUser method to be allowed access
	If retVal = 0 Then
		'User = Trim(InputBox("Enter User Name"))
		User = "JAR"  '*** TEST
		'Password = Trim(InputBox("Enter Password"))
		Password = "xxyzzy" '**** TEST
		retVal = oSS.nSetUser(User, Password)
	End If
	'If the SetUser fails, display LastErrorMsg and quit
	If retVal = 0 Then
		MsgBox (oSS.sLastErrorMsg & vbCrLf & "Quiting")
		'ERROR oSS.nCleanup()      ' Call Cleanup() before dropping the Session Object
		'ERROR oSS.DropObject()
		Set oSS = Nothing
		DoCmd.SetWarnings True
		Exit Sub
	End If
	'Set MAS 90 Company and Module
	'Company must be configured to Allow External Access
	retVal = oSS.nSetCompany("TST")     '**** TEST
	'If SetCompany fails, display LastErrorMsg and quit
	If retVal = 0 Then
		MsgBox (oSS.sLastErrorMsg & vbCrLf & "Quiting")
		'ERROR oSS.nCleanup()      ' Call Cleanup() before dropping the Session Object
		'ERROR oSS.DropObject()
		Set oSS = Nothing
		DoCmd.SetWarnings True
		Exit Sub
	End If
	retVal = oSS.nSetDate("A/P", "20200710")
	retVal = oSS.nSetModule("A/P")
	'MsgBox "Test " & pathhome & User & Password   '***** Test
	' Instantiate a A/P Data Endry business object
	oSEC = oSS.nSetProgram(oSS.nLookupTask("AP_Invoice_ui"))
	Set o = oScript.NewObject("AP_Invoice_bus", oSS)
	' Instantiate a AR_Vendor business object
	oSEC = oSS.nSetProgram(oSS.nLookupTask("AR_Vendor_ui"))
	Set oCust = oScript.NewObject("AP_Invoice_bus", oSS)
	Dim dbs As DAO.Database
	Dim rstMain As DAO.Recordset
	Dim rstLine As DAO.Recordset
	Set dbs = CurrentDb()
	SqlStr = "SELECT * FROM AP_Invoice_Export"   'what the hell
	Set rstMain = dbs.CreateQueryDef("", SqlStr).OpenRecordset
	Set rstLine = dbs.CreateQueryDef("", SqlStr).OpenRecordset
	'
	strHdrInvoiceNo = ""
	InvoiceDate = ""
	InvoiceDue = ""
	VendorNo = ""
	AP_BatchNo = ""
	'With rstMain
	rstMain.MoveFirst
	Do While Not rstMain.EOF
		strHdrInvoiceNo = rstMain!APDocNumber
		AP_BatchNo = Str(rstMain!APBatchID)
		If strHdrInvoiceNo <> prevInvoice Then
			strInvoiceNo = strHdrInvoiceNo
			InvoiceDate = "20200701" 'rstMain!APDocDate  ***TEST
			'OrderExp = rstMain!OrderExpDate
			'MsgBox "OrderDate " & OrderDate & " ExpDate " & OrderExp
			VendorNo = rstMain!CAccVendorID
			InvoiceAmt = rstMain!APExportAmount
			MsgBox "Header " & strInvoiceNo & " InvoiceDate " & InvoiceDate & " Vendor " & VendorNo & " Amount " & InvoiceAmt
			retVal = 0
			CName = ""
			custExist = 0
			InvoiceNo = ""
			prevInvoice = ""
			CustAdd1 = ""
			CustCity = ""
			CustCustNo = ""
			APDivisionNo = "00"
			LComment = ""
			' If batches are enabled for A/P Invoices create and use next new batch no.
			If o.nBatchEnabled = 1 Then
				o.nSelectbatch (AP_BatchNo)
			End If
			' Set the A/P Invoice Header record
			
			' ##### The following are not needed. #####
				' r = o.nSetKey() 
				' r = o.nSetValue("APDivisionNo$", APDivisionNo)
				' r = o.nSetValue("VendorNo$", VendorNo)
				' r = o.nSetValue("InvoiceNo$", strHdrInvoiceNo)
			' ##### The above are not needed. #####
			
			' ##### The following are changes i made/added. #####
				r = 0
				r = o.nSetKeyValue("APDivisionNo$", APDivisionNo)
				r = o.nSetKeyValue("VendorNo$", VendorNo)
				r = o.nSetKeyValue("InvoiceNo$", strHdrInvoiceNo)
			' ##### The above are changes i made. #####
			
			r = o.nSetKey()
			r = o.nSetValue("InvoiceDate$", InvoiceDate)
			r = o.nSetValue("InvoiceDueDate$", InvoiceDue)
			r = o.nSetValue("InvoiceAmt", InvoiceAmt)  'test
			prevInvoice = strHdrInvoiceNo
			' Create a new line for the item and write to memory file
			Do While Not rstLine.EOF
				LinesInvoiceno = rstLine!APDocNumber
				AcctKey = rstLine!GLAcctCode
				DistAmt = rstLine!Ext
				LComment = rstLine!APDOcItemDesc
				MsgBox "Invno" & LinesInvoiceno & " GL " & AcctKey & " DistAmt " & DistAmt & " Comment " & LComment
				If LinesInvoiceno = strHdrInvoiceNo Then
					' Create a new A/P Invoice Detail record
					r = o.oLines.nAddLine()
					' Set the column values for the detail record
					r = o.oLines.nSetValue("AccountKey$", AcctKey)
					r = o.oLines.nSetValue("DistributionAmt", DistAmt)
					'r = o.oLines.nSetValue("CommentText$", LComment)
					' Write the detail record (Memory file)
					
					' ##### The following are changes i made/added. #####
						r = o.oLines.nWrite()
					' ##### The above are changes i made. #####
					
				End If
				rstLine.MoveNext
			Loop
			' Write header and lines to disk
			r = o.nwrite()
			Me.Status = "Importing Invoice Number: " & InvoiceNo
		End If
		'Close Line file
		rstLine.Close
		Set rstLine = Nothing
		'
		'Open the Line file
		Set rstLine = dbs.CreateQueryDef("", SqlStr).OpenRecordset
		rstMain.MoveNext
	Loop
	MsgBox "Import Complete"
	rstMain.Close
	Set rstMain = Nothing
	Set dbs = Nothing
Else    'Import Question
End If
Me.Status = ""

Final thought, make sure the value in your AcctKey variable is actually the GL Account Key, not the formatted account number, if it is the latter, then you need to use the following instead to make use of the alternate index.

					r = o.oLines.nSetValue("AccountKey$", AcctKey, "kAccount")

I used your code and got the same result.  I had also made sure the AcctKey is the GL Account Key.

Are you sure the error is on the AddLine method?

Is it a hard error that halts execution?

Have you checked to make sure the o.oLines property is a valid object, perhaps it is not initialized yet.

If this is the case, you may need to instead use oSS.oNewObject([class name]) or oSS.oGetObject([class name]) instead of oScript.NewObject([class name], [session handle])

I believe it is.  I don't understand the last statement.   I don't know if this has anything to do with it, but I had to set the pathhome because they are running Sage 100 Preimum and there was no SOTAMAS90 in the CURRENT_USER registery

There are 3 different ways to start a new sage 100 object, all 3 are in my last post. You are currently using the last one i listed which uses the method from the ProvideX.Script COM object, i'm suggesting you try one of the first two that i listed which are methods of the SY_Session class object to see if that makes a difference.

Do you have any third party mods/extended solutions that have workstation dependencies?

I give up.  I'll go back to old VI.