IGXMLScanner2.cpp

                        XMLBufBid bbtemp(&fBufMgr);
                        XMLBuffer& tempBuf = bbtemp.getBuffer();

                        DatatypeValidator* tempDV = ((SchemaElementDecl*) topElem->fThisElement)->getDatatypeValidator();
                        ((SchemaValidator*) fValidator)->normalizeWhiteSpace(tempDV, toFill.getRawBuffer(),  tempBuf);
                        toFill.set(tempBuf.getRawBuffer());
                    }

                    // tell the schema validation about the character data for checkContent later
                    ((SchemaValidator*) fValidator)->setDatatypeBuffer(toFill.getRawBuffer());

                    // call all active identity constraints
                    unsigned int count = fMatcherStack->getMatcherCount();

                    for (unsigned int i = 0; i < count; i++) {
                        fMatcherStack->getMatcherAt(i)->docCharacters(toFill.getRawBuffer(), toFill.getLen());
                    }

                    if (fDocHandler)
                        fDocHandler->docCharacters(toFill.getRawBuffer(), toFill.getLen(), false);
                }
            }
            else
            {
                fValidator->emitError(XMLValid::NoCharDataInCM);
            }
        }
    }
    else
    {
        // call all active identity constraints
        if (fGrammarType == Grammar::SchemaGrammarType) {

            unsigned int count = fMatcherStack->getMatcherCount();

            for (unsigned int i = 0; i < count; i++) {
                fMatcherStack->getMatcherAt(i)->docCharacters(toSend.getRawBuffer(), toSend.getLen());
            }
        }

        // Always assume its just char data if not validating
        if (fDocHandler)
            fDocHandler->docCharacters(toSend.getRawBuffer(), toSend.getLen(), false);
    }

    // Reset buffer
    toSend.reset();
}



//  This method is called with a key/value string pair that represents an
//  xmlns="yyy" or xmlns:xxx="yyy" attribute. This method will update the
//  current top of the element stack based on this data. We know that when
//  we get here, that it is one of these forms, so we don't bother confirming
//  it.
//
//  But we have to ensure
//      1. xxx is not xmlns
//      2. if xxx is xml, then yyy must match XMLUni::fgXMLURIName, and vice versa
//      3. yyy is not XMLUni::fgXMLNSURIName
//      4. if xxx is not null, then yyy cannot be an empty string.
void IGXMLScanner::updateNSMap(const  XMLCh* const    attrName
                            , const XMLCh* const    attrValue)
{
    // We need a buffer to normalize the attribute value into
    XMLBufBid bbNormal(&fBufMgr);
    XMLBuffer& normalBuf = bbNormal.getBuffer();

    //  Normalize the value into the passed buffer. In this case, we don't
    //  care about the return value. An error was issued for the error, which
    //  is all we care about here.
    normalizeAttRawValue(attrName, attrValue, normalBuf);
    XMLCh* namespaceURI = normalBuf.getRawBuffer();

    //  We either have the default prefix (""), or we point it into the attr
    //  name parameter. Note that the xmlns is not the prefix we care about
    //  here. To us, the 'prefix' is really the local part of the attrName
    //  parameter.
    //
    //  Check 1. xxx is not xmlns
    //        2. if xxx is xml, then yyy must match XMLUni::fgXMLURIName, and vice versa
    //        3. yyy is not XMLUni::fgXMLNSURIName
    //        4. if xxx is not null, then yyy cannot be an empty string.
    const XMLCh* prefPtr = XMLUni::fgZeroLenString;
    const unsigned int colonOfs = XMLString::indexOf(attrName, chColon);
    if (colonOfs != -1) {
        prefPtr = &attrName[colonOfs + 1];

        if (XMLString::equals(prefPtr, XMLUni::fgXMLNSString))
            emitError(XMLErrs::NoUseOfxmlnsAsPrefix);
        else if (XMLString::equals(prefPtr, XMLUni::fgXMLString)) {
            if (!XMLString::equals(namespaceURI, XMLUni::fgXMLURIName))
                emitError(XMLErrs::PrefixXMLNotMatchXMLURI);
        }

        if (!namespaceURI || !*namespaceURI)
            emitError(XMLErrs::NoEmptyStrNamespace, attrName);
    }

    if (XMLString::equals(namespaceURI, XMLUni::fgXMLNSURIName))
        emitError(XMLErrs::NoUseOfxmlnsURI);
    else if (XMLString::equals(namespaceURI, XMLUni::fgXMLURIName)) {
        if (!XMLString::equals(prefPtr, XMLUni::fgXMLString))
            emitError(XMLErrs::XMLURINotMatchXMLPrefix);
    }

    //  Ok, we have to get the unique id for the attribute value, which is the
    //  URI that this value should be mapped to. The validator has the
    //  namespace string pool, so we ask him to find or add this new one. Then
    //  we ask the element stack to add this prefix to URI Id mapping.
    fElemStack.addPrefix
    (
        prefPtr
        , fURIStringPool->addOrFind(namespaceURI)
    );
}

void IGXMLScanner::scanRawAttrListforNameSpaces(const RefVectorOf<KVStringPair>* theRawAttrList, int attCount)
{
    //  Make an initial pass through the list and find any xmlns attributes or
    //  schema attributes.
    //  When we find one, send it off to be used to update the element stack's
    //  namespace mappings.
    int index = 0;
    for (index = 0; index < attCount; index++)
    {
        // each attribute has the prefix:suffix="value"
        const KVStringPair* curPair = fRawAttrList->elementAt(index);
        const XMLCh* rawPtr = curPair->getKey();

        //  If either the key begins with "xmlns:" or its just plain
        //  "xmlns", then use it to update the map.
        if (!XMLString::compareNString(rawPtr, XMLUni::fgXMLNSColonString, 6)
        ||  XMLString::equals(rawPtr, XMLUni::fgXMLNSString))
        {
            const XMLCh* valuePtr = curPair->getValue();

            updateNSMap(rawPtr, valuePtr);

            // if the schema URI is seen in the the valuePtr, set the boolean seeXsi
            if (XMLString::equals(valuePtr, SchemaSymbols::fgURI_XSI)) {
                fSeeXsi = true;
            }
        }
    }

    // walk through the list again to deal with "xsi:...."
    if (fDoSchema && fSeeXsi)
    {
        //  Schema Xsi Type yyyy (e.g. xsi:type="yyyyy")
        XMLBufBid bbXsi(&fBufMgr);
        XMLBuffer& fXsiType = bbXsi.getBuffer();

        QName attName;

        for (index = 0; index < attCount; index++)
        {
            // each attribute has the prefix:suffix="value"
            const KVStringPair* curPair = fRawAttrList->elementAt(index);
            const XMLCh* rawPtr = curPair->getKey();

            attName.setName(rawPtr, fEmptyNamespaceId);
            const XMLCh* prefPtr = attName.getPrefix();

            // if schema URI has been seen, scan for the schema location and uri
            // and resolve the schema grammar; or scan for schema type
            if (resolvePrefix(prefPtr, ElemStack::Mode_Attribute) == fSchemaNamespaceId) {

                const XMLCh* valuePtr = curPair->getValue();
                const XMLCh* suffPtr = attName.getLocalPart();

                if (XMLString::equals(suffPtr, SchemaSymbols::fgXSI_SCHEMALOCACTION))
                    parseSchemaLocation(valuePtr);
                else if (XMLString::equals(suffPtr, SchemaSymbols::fgXSI_NONAMESPACESCHEMALOCACTION))
                    resolveSchemaGrammar(valuePtr, XMLUni::fgZeroLenString);

                if (XMLString::equals(suffPtr, SchemaSymbols::fgXSI_TYPE)) {
                        fXsiType.set(valuePtr);
                }
                else if (XMLString::equals(suffPtr, SchemaSymbols::fgATT_NILL)
                         && fValidator && fValidator->handlesSchema()
                         && XMLString::equals(valuePtr, SchemaSymbols::fgATTVAL_TRUE)) {
                            ((SchemaValidator*)fValidator)->setNillable(true);
                }
            }
        }

        if (fValidator && fValidator->handlesSchema()) {
            if (!fXsiType.isEmpty()) {
                int colonPos = -1;
                unsigned int uriId = resolveQName (
                      fXsiType.getRawBuffer()
                    , fPrefixBuf
                    , ElemStack::Mode_Element
                    , colonPos
                );
                ((SchemaValidator*)fValidator)->setXsiType(fPrefixBuf.getRawBuffer(), fXsiType.getRawBuffer() + colonPos + 1, uriId);
            }
        }
    }
}

void IGXMLScanner::parseSchemaLocation(const XMLCh* const schemaLocationStr)
{
    BaseRefVectorOf<XMLCh>* schemaLocation = XMLString::tokenizeString(schemaLocationStr);
    unsigned int size = schemaLocation->size();
    if (size % 2 != 0 ) {
        emitError(XMLErrs::BadSchemaLocation);
    } else {
        for(unsigned int i=0; i<size; i=i+2) {
            resolveSchemaGrammar(schemaLocation->elementAt(i+1), schemaLocation->elementAt(i));
        }
    }

    delete schemaLocation;
}

void IGXMLScanner::resolveSchemaGrammar(const XMLCh* const loc, const XMLCh* const uri) {

    Grammar* grammar = fGrammarResolver->getGrammar(uri);

    if (!grammar || grammar->getGrammarType() == Grammar::DTDGrammarType) {
        XSDDOMParser parser;

        parser.setValidationScheme(XercesDOMParser::Val_Never);
        parser.setDoNamespaces(true);
        parser.setUserEntityHandler(fEntityHandler);
        parser.setUserErrorReporter(fErrorReporter);

        // Create a buffer for expanding the system id
        XMLBufBid bbSys(&fBufMgr);
        XMLBuffer& expSysId = bbSys.getBuffer();
        XMLBuffer& normalizedSysId = bbSys.getBuffer();

        normalizeURI(loc, normalizedSysId);

        //  Allow the entity handler to expand the system id if they choose
        //  to do so.
        InputSource* srcToFill = 0;
        const XMLCh* normalizedURI = normalizedSysId.getRawBuffer();
        if (fEntityHandler)
        {
            if (!fEntityHandler->expandSystemId(normalizedURI, expSysId))
                expSysId.set(normalizedURI);

            srcToFill = fEntityHandler->resolveEntity( XMLUni::fgZeroLenString
                                                     , expSysId.getRawBuffer());
        }
        else
        {
            expSysId.set(normalizedURI);
        }

        //  If they didn't create a source via the entity handler, then we
        //  have to create one on our own.
        if (!srcToFill)
        {
            ReaderMgr::LastExtEntityInfo lastInfo;
            fReaderMgr.getLastExtEntityInfo(lastInfo);

            try
            {
                XMLURL urlTmp(lastInfo.systemId, expSysId.getRawBuffer());
                if (urlTmp.isRelative())
                {
                    ThrowXML
                    (
                        MalformedURLException
                        , XMLExcepts::URL_NoProtocolPresent
                    );
                }
                srcToFill = new URLInputSource(urlTmp);
            }

            catch(const MalformedURLException&)
            {
                // Its not a URL, so lets assume its a local file name.
                srcToFill = new LocalFileInputSource
                (
                    lastInfo.systemId
                    , expSysId.getRawBuffer()
                );
            }
        }

        // Put a janitor on the input source
        Janitor<InputSource> janSrc(srcToFill);

        // Should just issue warning if the schema is not found
        const bool flag = srcToFill->getIssueFatalErrorIfNotFound();
        srcToFill->setIssueFatalErrorIfNotFound(false);

        parser.parse(*srcToFill);

        // Reset the InputSource
        srcToFill->setIssueFatalErrorIfNotFound(flag);

        if (parser.getSawFatal() && fExitOnFirstFatal)
            emitError(XMLErrs::SchemaScanFatalError);

        DOMDocument* document = parser.getDocument(); //Our Grammar

        if (document != 0) {

            DOMElement* root = document->getDocumentElement();// This is what we pass to TraverserSchema
            if (root != 0)
            {
                const XMLCh* newUri = root->getAttribute(SchemaSymbols::fgATT_TARGETNAMESPACE);
                if (!XMLString::equals(newUri, uri)) {
                    if (fValidate)
                        fValidator->emitError(XMLValid::WrongTargetNamespace, loc, uri);
                    grammar = fGrammarResolver->getGrammar(newUri);
                }

                if (!grammar || grammar->getGrammarType() == Grammar::DTDGrammarType) {

                    //  Since we have seen a grammar, set our validation flag
                    //  at this point if the validation scheme is auto
                    if (fValScheme == Val_Auto && !fValidate) {
                        fValidate = true;
                        fElemStack.setValidationFlag(fValidate);
                    }

                    // we have seen a schema, so set up the fValidator as fSchemaValidator
                    if (!fValidator->handlesSchema())
                    {
                        if (fValidatorFromUser) {
                            // the fValidator is from user
                            ThrowXML(RuntimeException, XMLExcepts::Gen_NoSchemaValidator);
                        }
                        else {
                            fValidator = fSchemaValidator;
                        }
                    }

                    grammar = new SchemaGrammar();
                    TraverseSchema traverseSchema(root, fURIStringPool, (SchemaGrammar*) grammar, fGrammarResolver, this, srcToFill->getSystemId(), fEntityHandler, fErrorReporter);

                    if (fGrammarType == Grammar::DTDGrammarType) {
                        fGrammar = grammar;
                        fGrammarType = Grammar::SchemaGrammarType;
                        fValidator->setGrammar(fGrammar);
                    }

                    if (fValidate) {
                        //  validate the Schema scan so far
                        fValidator->preContentValidation(false);
                    }
                }
            }
        }
    }
    else {

        //  Since we have seen a grammar, set our validation flag
        //  at this point if the validation scheme is auto
        if (fValScheme == Val_Auto && !fValidate) {
            fValidate = true;
            fElemStack.setValidationFlag(fValidate);
        }

        // we have seen a schema, so set up the fValidator as fSchemaValidator
        if (!fValidator->handlesSchema())
        {
            if (fValidatorFromUser) {
                // the fValidator is from user
                ThrowXML(RuntimeException, XMLExcepts::Gen_NoSchemaValidator);
            }
            else {
                fValidator = fSchemaValidator;
            }
        }

        if (fGrammarType == Grammar::DTDGrammarType) {
            fGrammar = grammar;
            fGrammarType = Grammar::SchemaGrammarType;
            fValidator->setGrammar(fGrammar);
        }
    }
}

InputSource* IGXMLScanner::resolveSystemId(const XMLCh* const sysId)
{
    // Create a buffer for expanding the system id
    XMLBufBid bbSys(&fBufMgr);
    XMLBuffer& expSysId = bbSys.getBuffer();

    //  Allow the entity handler to expand the system id if they choose
    //  to do so.
    InputSource* srcToFill = 0;
    if (fEntityHandler)
    {
        if (!fEntityHandler->expandSystemId(sysId, expSysId))
            expSysId.set(sysId);

        srcToFill = fEntityHandler->resolveEntity( XMLUni::fgZeroLenString
                                                 , expSysId.getRawBuffer());
    }
    else
    {
        expSysId.set(sysId);
    }

    //  If they didn't create a source via the entity handler, then we
    //  have to create one on our own.
    if (!srcToFill)
    {
        ReaderMgr::LastExtEntityInfo lastInfo;
        fReaderMgr.getLastExtEntityInfo(lastInfo);

        try
        {
            XMLURL urlTmp(lastInfo.systemId, expSysId.getRawBuffer());
            if (urlTmp.isRelative())
            {
                ThrowXML
                (
                    MalformedURLException
                    , XMLExcepts::URL_NoProtocolPresent
                );
            }
            srcToFill = new URLInputSource(urlTmp);
        }
        catch(const MalformedURLException&)
        {
            // Its not a URL, so lets assume its a local file name.
            srcToFill = new LocalFileInputSource
            (
                lastInfo.systemId
                , expSysId.getRawBuffer()
            );
        }
    }

    return srcToFill;
}


// ---------------------------------------------------------------------------
//  IGXMLScanner: Private grammar preparsing methods
// ---------------------------------------------------------------------------
Grammar* IGXMLScanner::loadXMLSchemaGrammar(const InputSource& src,
                                          const bool toCache)
{
   // Reset the validators
    fSchemaValidator->reset();
    fSchemaValidator->setErrorReporter(fErrorReporter);
    fSchemaValidator->setExitOnFirstFatal(fExitOnFirstFatal);

    if (fValidatorFromUser)
        fValidator->reset();

    if (!fValidator->handlesSchema()) {
        if (fValidatorFromUser && fValidate)
            ThrowXML(RuntimeException, XMLExcepts::Gen_NoSchemaValidator);
        else {
            fValidator = fSchemaValidator;
        }
    }

    XSDDOMParser parser;

    parser.setValidationScheme(XercesDOMParser::Val_Never);
    parser.setDoNamespaces(true);
    parser.setUserEntityHandler(fEntityHandler);
    parser.setUserErrorReporter(fErrorReporter);

    // Should just issue warning if the schema is not found
    const bool flag = src.getIssueFatalErrorIfNotFound();
    ((InputSource&) src).setIssueFatalErrorIfNotFound(false);

    parser.parse(src);

    // Reset the InputSource
    ((InputSource&) src).setIssueFatalErrorIfNotFound(flag);

    if (parser.getSawFatal() && fExitOnFirstFatal)
        emitError(XMLErrs::SchemaScanFatalError);

    DOMDocument* document = parser.getDocument(); //Our Grammar

    if (document != 0) {

        DOMElement* root = document->getDocumentElement();// This is what we pass to TraverserSchema
        if (root != 0)
        {
            SchemaGrammar* grammar = new SchemaGrammar();
            TraverseSchema traverseSchema(root, fURIStringPool, (SchemaGrammar*) grammar, fGrammarResolver, this, src.getSystemId(), fEntityHandler, fErrorReporter);

            if (fValidate) {
                //  validate the Schema scan so far
                fValidator->setGrammar(grammar);
                fValidator->preContentValidation(false, true);
            }

            if (toCache) {
                fGrammarResolver->cacheGrammars();
            }

            return grammar;
        }
    }

    return 0;
}



// ---------------------------------------------------------------------------
//  IGXMLScanner: Private parsing methods
// ---------------------------------------------------------------------------

//  This method is called to do a raw scan of an attribute value. It does not
//  do normalization (since we don't know their types yet.) It just scans the
//  value and does entity expansion.
//
//  End of entity's must be dealt with here. During DTD scan, they can come
//  from external entities. During content, they can come from any entity.
//  We just eat the end of entity and continue with our scan until we come
//  to the closing quote. If an unterminated value causes us to go through
//  subsequent entities, that will cause errors back in the calling code,
//  but there's little we can do about it here.
bool IGXMLScanner::basicAttrValueScan(const XMLCh* const attrName, XMLBuffer& toFill)
{
    // Reset the target buffer
    toFill.reset();

    // Get the next char which must be a single or double quote
    XMLCh quoteCh;
    if (!fReaderMgr.skipIfQuote(quoteCh))
        return false;

    //  We have to get the current reader because we have to ignore closing
    //  quotes until we hit the same reader again.
    const unsigned int curReader = fReaderMgr.getCurrentReaderNum();

    //  Loop until we get the attribute value. Note that we use a double
    //  loop here to avoid the setup/teardown overhead of the exception
    //  handler on every round.
    XMLCh   nextCh;
    XMLCh   secondCh = 0;
    bool    gotLeadingSurrogate = false;
    bool    escaped;
    bool    charref_expanded = false;
    while (true)
    {
        try
        {
            while(true)
            {
                // Get another char. Use second char if one is waiting
                if (secondCh)
                {
                    nextCh = secondCh;
                    secondCh = 0;
                }
                else
                {
                    nextCh = fReaderMgr.getNextChar();
                }

                if (!nextCh)
                    ThrowXML(UnexpectedEOFException, XMLExcepts::Gen_UnexpectedEOF);

                //  Check for our ending quote. It has to be in the same entity
                //  as where we started. Quotes in nested entities are ignored.
                if (nextCh == quoteCh)
                {
                    if (curReader == fReaderMgr.getCurrentReaderNum())
                        return true;

                    // Watch for spillover into a previous entity
                    if (curReader > fReaderMgr.getCurrentReaderNum())
                    {
                        emitError(XMLErrs::PartialMarkupInEntity);
                        return false;
                    }
                }

                //  Check for an entity ref . We ignore the empty flag in
                //  this one.
                escaped = false;
                if (nextCh == chAmpersand)
                {
                    // If it was not returned directly, then jump back up
                    if (scanEntityRef(true, nextCh, secondCh, escaped) != EntityExp_Returned)
                    {
                        gotLeadingSurrogate = false;
                        continue;
                    }
                    charref_expanded = true;
                }

                // Deal with surrogate pairs
                if ((nextCh >= 0xD800) && (nextCh <= 0xDBFF))
                {
                    //  Its a leading surrogate. If we already got one, then
                    //  issue an error, else set leading flag to make sure that
                    //  we look for a trailing next time.
                    if (gotLeadingSurrogate)
                    {
                        emitError(XMLErrs::Expected2ndSurrogateChar);
                    }
                    else
                        gotLeadingSurrogate = true;
                }
                else
                {
                    //  If its a trailing surrogate, make sure that we are
                    //  prepared for that. Else, its just a regular char so make
                    //  sure that we were not expected a trailing surrogate.
                    if ((nextCh >= 0xDC00) && (nextCh <= 0xDFFF))
                    {
                        // Its trailing, so make sure we were expecting it
                        if (!gotLeadingSurrogate)
                            emitError(XMLErrs::Unexpected2ndSurrogateChar);
                    }
                    else
                    {
                        //  Its just a char, so make sure we were not expecting a
                        //  trailing surrogate.
                        if (gotLeadingSurrogate) {
                            emitError(XMLErrs::Expected2ndSurrogateChar);
                        }
                        // Its got to at least be a valid XML character
                        else if (!fReaderMgr.getCurrentReader()->isXMLChar(nextCh))
                        {
                            // if it was a character reference and is control char, then it's ok
                            if (!(charref_expanded && fReaderMgr.getCurrentReader()->isControlChar(nextCh)))
                            {

                                XMLCh tmpBuf[9];
                                XMLString::binToText
                                (
                                    nextCh
                                    , tmpBuf
                                    , 8
                                    , 16
                                );
                                emitError(XMLErrs::InvalidCharacterInAttrValue, attrName, tmpBuf);
                            }
                        }
                    }
                    charref_expanded = false;
                    gotLeadingSurrogate = false;
                }

                //  If it was escaped, then put in a 0xFFFF value. This will
                //  be used later during validation and normalization of the
                //  value to know that the following character was via an
                //  escape char.
                if (escaped)
                    toFill.append(0xFFFF);

                // Else add it to the buffer
                toFill.append(nextCh);
            }
        }
        catch(const EndOfEntityException&)
        {
            // Just eat it and continue.
            gotLeadingSurrogate = false;
            escaped = false;
        }
    }
    return true;
}


bool IGXMLScanner::scanAttValue(  const   XMLAttDef* const    attDef
                                  ,       XMLBuffer&          toFill)
{
    enum States
    {
        InWhitespace
        , InContent
    };

    // Get the type and name
    const XMLAttDef::AttTypes type = attDef->getType();
    const XMLCh* const attrName = attDef->getFullName();

    // Reset the target buffer
    toFill.reset();

    // Get the next char which must be a single or double quote
    XMLCh quoteCh;
    if (!fReaderMgr.skipIfQuote(quoteCh))
        return false;

    //  We have to get the current reader because we have to ignore closing
    //  quotes until we hit the same reader again.
    const unsigned int curReader = fReaderMgr.getCurrentReaderNum();

    // Get attribute def - to check to see if it's declared externally or not
    bool  isAttExternal = attDef->isExternal();

    //  Loop until we get the attribute value. Note that we use a double
    //  loop here to avoid the setup/teardown overhead of the exception
    //  handler on every round.
    XMLCh   nextCh;
    XMLCh   secondCh = 0;
    States  curState = InContent;
    bool    firstNonWS = false;
    bool    gotLeadingSurrogate = false;
    bool    escaped;
    bool    charref_expanded = false;
    while (true)
    {
    try
    {
        while(true)
        {
            // Get another char. Use second char if one is waiting
            if (secondCh)
            {
                nextCh = secondCh;
                secondCh = 0;
            }
            else
            {
                nextCh = fReaderMgr.getNextChar();
            }

            if (!nextCh)
                ThrowXML(UnexpectedEOFException, XMLExcepts::Gen_UnexpectedEOF);

            // Check for our ending quote in the same entity
            if (nextCh == quoteCh)
            {
                if (curReader == fReaderMgr.getCurrentReaderNum())
                    return true;

                // Watch for spillover into a previous entity
                if (curReader > fReaderMgr.getCurrentReaderNum())
                {
                    emitError(XMLErrs::PartialMarkupInEntity);
                    return false;
                }
            }

            //  Check for an entity ref now, before we let it affect our
            //  whitespace normalization logic below. We ignore the empty flag
            //  in this one.
            escaped = false;
            if (nextCh == chAmpersand)
            {
                if (scanEntityRef(true, nextCh, secondCh, escaped) != EntityExp_Returned)
                {
                    gotLeadingSurrogate = false;
                    continue;
                }
                charref_expanded = true;
            }

            // Deal with surrogate pairs
            if ((nextCh >= 0xD800) && (nextCh <= 0xDBFF))
            {
                //  Its a leading surrogate. If we already got one, then
                //  issue an error, else set leading flag to make sure that
                //  we look for a trailing next time.
                if (gotLeadingSurrogate)
                    emitError(XMLErrs::Expected2ndSurrogateChar);
                 else
                    gotLeadingSurrogate = true;
            }
            else
            {
                //  If its a trailing surrogate, make sure that we are
                //  prepared for that. Else, its just a regular char so make
                //  sure that we were not expected a trailing surrogate.
                if ((nextCh >= 0xDC00) && (nextCh <= 0xDFFF))
                {
                    // Its trailing, so make sure we were expecting it
                    if (!gotLeadingSurrogate)
                        emitError(XMLErrs::Unexpected2ndSurrogateChar);
                }
                else
                {
                    //  Its just a char, so make sure we were not expecting a
                    //  trailing surrogate.
                    if (gotLeadingSurrogate)
                        emitError(XMLErrs::Expected2ndSurrogateChar);

                    // Its got to at least be a valid XML character
                    if (!fReaderMgr.getCurrentReader()->isXMLChar(nextCh))
                    {
                        // if it was a character reference and is control char, then it's ok
                        if (!(charref_expanded && fReaderMgr.getCurrentReader()->isControlChar(nextCh)))
                        {
                            XMLCh tmpBuf[9];
                            XMLString::binToText
                            (
                                nextCh
                                , tmpBuf
                                , 8
                                , 16
                            );
                            emitError(XMLErrs::InvalidCharacterInAttrValue, attrName, tmpBuf);
                        }
                    }
                }
                charref_expanded = false;
                gotLeadingSurrogate = false;
            }

            //  If its not escaped, then make sure its not a < character, which
            //  is not allowed in attribute values.
            if (!escaped && (nextCh == chOpenAngle))
                emitError(XMLErrs::BracketInAttrValue, attrName);

            //  If the attribute is a CDATA type we do simple replacement of
            //  tabs and new lines with spaces, if the character is not escaped
            //  by way of a char ref.
            //
            //  Otherwise, we do the standard non-CDATA normalization of
            //  compressing whitespace to single spaces and getting rid of leading
            //  and trailing whitespace.
            if (type == XMLAttDef::CData)
            {
                if (!escaped)
                {
                    if ((nextCh == 0x09) || (nextCh == 0x0A) || (nextCh == 0x0D))
                    {
                        // Check Validity Constraint for Standalone document declaration
                        // XML 1.0, Section 2.9
                        if (fStandalone && fValidate && isAttExternal)
                        {
                             // Can't have a standalone document declaration of "yes" if  attribute
                             // values are subject to normalisation
                             fValidator->emitError(XMLValid::NoAttNormForStandalone, attrName);
                        }
                        nextCh = chSpace;
                    }
                }
            }
            else
            {
                if (curState == InWhitespace)
                {
                    if ((escaped && nextCh != chSpace) || !fReaderMgr.getCurrentReader()->isWhitespace(nextCh))
                    {
                        if (firstNonWS)
                            toFill.append(chSpace);
                        curState = InContent;
                        firstNonWS = true;
                    }
                    else
                    {
                        continue;
                    }
                }
                else if (curState == InContent)
                {
                    if ((nextCh == chSpace) ||
                        (fReaderMgr.getCurrentReader()->isWhitespace(nextCh) && !escaped))
                    {
                        curState = InWhitespace;

                        // Check Validity Constraint for Standalone document declaration
                        // XML 1.0, Section 2.9
                        if (fStandalone && fValidate && isAttExternal)
                        {
                            if (!firstNonWS || (nextCh != chSpace) || (fReaderMgr.lookingAtSpace()))
                            {
                                 // Can't have a standalone document declaration of "yes" if  attribute
                                 // values are subject to normalisation
                                 fValidator->emitError(XMLValid::NoAttNormForStandalone, attrName);
                            }
                        }
                        continue;
                    }
                    firstNonWS = true;
                }
            }

            // Else add it to the buffer
            toFill.append(nextCh);
        }
    }
    catch(const EndOfEntityException&)
    {
        // Just eat it and continue.
        gotLeadingSurrogate = false;
        escaped = false;
    }
    }
    return true;
}


//  This method scans a CDATA section. It collects the character into one
//  of the temp buffers and calls the document handler, if any, with the
//  characters. It assumes that the <![CDATA string has been scanned before
//  this call.
void IGXMLScanner::scanCDSection()
{
    //  This is the CDATA section opening sequence, minus the '<' character.
    //  We use this to watch for nested CDATA sections, which are illegal.
    static const XMLCh CDataPrefix[] =
    {
            chBang, chOpenSquare, chLatin_C, chLatin_D, chLatin_A
        ,   chLatin_T, chLatin_A, chOpenSquare, chNull
    };

    static const XMLCh CDataClose[] =
    {
            chCloseSquare, chCloseAngle, chNull
    };

    //  The next character should be the opening square bracket. If not
    //  issue an error, but then try to recover by skipping any whitespace
    //  and checking again.
    if (!fReaderMgr.skippedChar(chOpenSquare))
    {
        emitError(XMLErrs::ExpectedOpenSquareBracket);
        fReaderMgr.skipPastSpaces();

        // If we still don't find it, then give up, else keep going
        if (!fReaderMgr.skippedChar(chOpenSquare))
            return;
    }

    // Get a buffer for this
    XMLBufBid bbCData(&fBufMgr);

    //  We just scan forward until we hit the end of CDATA section sequence.
    //  CDATA is effectively a big escape mechanism so we don't treat markup
    //  characters specially here.
    bool            emittedError = false;
    bool    gotLeadingSurrogate = false;
    while (true)
    {
        const XMLCh nextCh = fReaderMgr.getNextChar();

        // Watch for unexpected end of file
        if (!nextCh)
        {
            emitError(XMLErrs::UnterminatedCDATASection);
            ThrowXML(UnexpectedEOFException, XMLExcepts::Gen_UnexpectedEOF);
        }

        if (fValidate && fStandalone && (fReaderMgr.getCurrentReader()->isWhitespace(nextCh)))
        {
            // This document is standalone; this ignorable CDATA whitespace is forbidden.
            // XML 1.0, Section 2.9
            // And see if the current element is a 'Children' style content model
            const ElemStack::StackElem* topElem = fElemStack.topElement();

            if (topElem->fThisElement->isExternal()) {

                // Get the character data opts for the current element
                XMLElementDecl::CharDataOpts charOpts =  topElem->fThisElement->getCharDataOpts();

                if (charOpts == XMLElementDecl::SpacesOk) // Element Content
                {
                    // Error - standalone should have a value of "no" as whitespace detected in an
                    // element type with element content whose element declaration was external
                    fValidator->emitError(XMLValid::NoWSForStandalone);
                }
            }
        }

        //  If this is a close square bracket it could be our closing
        //  sequence.
        if (nextCh == chCloseSquare && fReaderMgr.skippedString(CDataClose))
        {
            if (fGrammarType == Grammar::SchemaGrammarType) {

                // call all active identity constraints
                unsigned int count = fMatcherStack->getMatcherCount();

                for (unsigned int i = 0; i < count; i++) {
                    fMatcherStack->getMatcherAt(i)->docCharacters(bbCData.getRawBuffer(), bbCData.getLen());
                }
            }

            // If we have a doc handler, call it
            if (fDocHandler)
            {
                fDocHandler->docCharacters
                    (
                    bbCData.getRawBuffer()
                    , bbCData.getLen()
                    , true
                    );
            }

            // And we are done
            break;
        }

        //  Make sure its a valid character. But if we've emitted an error
        //  already, don't bother with the overhead since we've already told
        //  them about it.
        if (!emittedError)
        {
            // Deal with surrogate pairs
            if ((nextCh >= 0xD800) && (nextCh <= 0xDBFF))