You need to define your function before you call it.

Declaring it outside of beforeSubmit keeps things nice and neat:

/**
 *@NApiVersion 2.x
 *@NScriptType UserEventScript
 */
define(['N/record', 'N/email', 'N/url', 'N/runtime', 'N/search', 'N/ui/serverWidget'],
function(record, email, url, runtime, search, serverWidget) {
    function CheckDuplicate(){
        var res=[];
        var customrecord_gbs_tax_rule_expt_nexusSearchObj = search.create({
            type: "customrecord_gbs_tax_rule_expt_nexus",
            filters:
                [
                ],
            columns:
                [
                    search.createColumn({name: "internalid", label: "Internal ID"}),
                    search.createColumn({name: "custrecord_gbs_tren_expt_nexus", label: "Exempt Nexus"})
                ]
        });
        var searchResultCount = customrecord_gbs_tax_rule_expt_nexusSearchObj.runPaged().count;
        //log.debug("expensereportSearchObj result count",searchResultCount);
        // alert('Expenses Count: '+searchResultCount);
        expensereportSearchObj.run().each(function(result){
            // .run().each has a limit of 4,000 results
            var category=result.getValue({name:"expensedate"});
            var exemptNexus =result.getValue({name:"expensecategory"});
            //if(result.id != recordId){// Exclude the current record
            res.push({
                'category':category,
                'exemptNexus':exemptNexus
            });
            //}
            return true;
        });
        return res;
    }