Out of memory error while attempting to load document to XML DB

classic Classic list List threaded Threaded
6 messages Options
Reply | Threaded
Open this post in threaded view
|

Out of memory error while attempting to load document to XML DB

Mahender Didwania
Hi,

I fetch some documents from VCS and load them to an XML DB.  The guide code is below.  I am presently getting out of java heap space error when attempting to load only two documents in total (it succeeds with the smaller document, 631 KB in size, but fails with the larger document, which is only 3.5 MB in size.  I am using Orbeon 3.8 on apache tomcat (xampp) on a Windows XP machine with 2 gigs of memory (got 800 MB physical RAM free when Tomcat is active).  File which is loaded using url-generator processor (not shown here) and made available on #configuration input is merely 2 KB in size.

Any pointers as to what I might be doing wrong?

  <p:for-each href="#configuration" select="/Configuration/Regions/Region" id="output" ref="data" root="results">
    <p:processor name="oxf:identity">
      <p:input name="data" href="current()"/>
      <p:output name="data" id="regionBeingProcessed"/>
    </p:processor>
    <p:for-each href="current()" select="/Region/Schema" id="schemaOutput" ref="output" root="resultsSchema">
      <p:processor name="oxf:identity">
        <p:input name="data" href="current()"/>
        <p:output name="data" id="schemaBeingProcessed"/>
      </p:processor>
      <p:for-each href="current()" select="/Schema/Data" id="dataOutput" ref="schemaOutput" root="resultsData">
        <p:processor name="oxf:identity">
          <p:input name="data" href="current()"/>
          <p:output name="data" id="dataBeingProcessed"/>
        </p:processor>
        <p:processor name="oxf:xslt">
          <p:input name="config">
            <xsl:stylesheet version="2.0">
              <xsl:template match="/">
                <TargetURL>
                  <xsl:variable name="location" select="/Configuration/XMLDB/Location/text()"/>              
                  <xsl:variable name="name" select="/Configuration/XMLDB/Name/text()"/>              
                  <xsl:variable name="region" select="doc('input:regionBeingProcessed')/Region/@id"/>
                  <xsl:variable name="dataSet" select="doc('input:dataBeingProcessed')/Data/@id"/>              
                  <xsl:value-of select="fn:concat($location,'/',$name,'/',$dataSet,$region)"/>
                </TargetURL>
              </xsl:template>
            </xsl:stylesheet>
          </p:input>
          <p:input name="data" href="#configuration"/>
          <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
          <p:input name="dataBeingProcessed" href="#dataBeingProcessed"/>
          <p:output name="data" id="TargetURL"/>
        </p:processor>
        <p:for-each href="current()" select="/Data/VCSPath" id="output2" ref="dataOutput" root="results">
          <p:processor name="oxf:xslt">
            <p:input name="config">
              <xsl:stylesheet version="2.0">
                <xsl:template match="/">
                  <SourceURL>
                    <xsl:variable name="locationPath" select="/Configuration/VCS/BrowserLocation/text()"/>              
                    <xsl:variable name="regionPath" select="doc('input:regionBeingProcessed')/Region/VCSPath/text()"/>
                    <xsl:variable name="schemaPath" select="doc('input:schemaBeingProcessed')/Schema/VCSPath/text()"/>
                    <xsl:variable name="namePath" select="doc('input:fileBeingProcessed')/VCSPath/text()"/>              
                    <xsl:value-of select="fn:string-join(($locationPath,$regionPath,$schemaPath,$namePath), '/')"/>
                  </SourceURL>
                </xsl:template>
              </xsl:stylesheet>
            </p:input>
            <p:input name="data" href="#configuration"/>
            <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
            <p:input name="schemaBeingProcessed" href="#schemaBeingProcessed"/>
            <p:input name="fileBeingProcessed" href="current()"/>
            <p:output name="data" id="SourceURL"/>
          </p:processor>
          <p:processor name="oxf:xforms-submission">
            <p:input name="submission">
              <xforms:submission method="get" action="{/SourceURL/text()}" mediatype="text/xml;" serialization="none">
                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
                  <xforms:output value="event('error-type')"/>
                </xforms:message>
              </xforms:submission>
            </p:input>
            <p:input name="request" href="#SourceURL"/>
            <p:output name="response" id="xml"/>
          </p:processor>
          <p:processor name="oxf:xforms-submission">
            <p:input name="submission" transform="oxf:xslt" href="#TargetURL">
              <xforms:submission xsl:version="2.0" method="post" action="{/TargetURL/text()}" xxforms:username="admin" xxforms:password="admin"  mediatype="text/xml;">
                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
                  <xforms:output value="event('error-type')"/>
                </xforms:message>
              </xforms:submission>
            </p:input>
            <p:input name="request" href="#xml"/>
            <p:output name="response" ref="output2"/>
          </p:processor>
        </p:for-each>
      </p:for-each>
    </p:for-each>
  </p:for-each>
 


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws
Reply | Threaded
Open this post in threaded view
|

Re: Out of memory error while attempting to load document to XML DB

Erik Bruchez
Administrator
So what's the size of your Java heap?

I would expect a 3.5 MB XML file to translate into at least 10x that
in RAM. Multiply by a factor if the document does through "tees" in
the pipeline as the document will then be stored in memory possibly
more than once.

-Erik

On Wed, Dec 15, 2010 at 6:57 AM, Mahender Didwania
<[hidden email]> wrote:

> Hi,
>
> I fetch some documents from VCS and load them to an XML DB.  The guide code is below.  I am presently getting out of java heap space error when attempting to load only two documents in total (it succeeds with the smaller document, 631 KB in size, but fails with the larger document, which is only 3.5 MB in size.  I am using Orbeon 3.8 on apache tomcat (xampp) on a Windows XP machine with 2 gigs of memory (got 800 MB physical RAM free when Tomcat is active).  File which is loaded using url-generator processor (not shown here) and made available on #configuration input is merely 2 KB in size.
>
> Any pointers as to what I might be doing wrong?
>
>  <p:for-each href="#configuration" select="/Configuration/Regions/Region" id="output" ref="data" root="results">
>    <p:processor name="oxf:identity">
>      <p:input name="data" href="current()"/>
>      <p:output name="data" id="regionBeingProcessed"/>
>    </p:processor>
>    <p:for-each href="current()" select="/Region/Schema" id="schemaOutput" ref="output" root="resultsSchema">
>      <p:processor name="oxf:identity">
>        <p:input name="data" href="current()"/>
>        <p:output name="data" id="schemaBeingProcessed"/>
>      </p:processor>
>      <p:for-each href="current()" select="/Schema/Data" id="dataOutput" ref="schemaOutput" root="resultsData">
>        <p:processor name="oxf:identity">
>          <p:input name="data" href="current()"/>
>          <p:output name="data" id="dataBeingProcessed"/>
>        </p:processor>
>        <p:processor name="oxf:xslt">
>          <p:input name="config">
>            <xsl:stylesheet version="2.0">
>              <xsl:template match="/">
>                <TargetURL>
>                  <xsl:variable name="location" select="/Configuration/XMLDB/Location/text()"/>
>                  <xsl:variable name="name" select="/Configuration/XMLDB/Name/text()"/>
>                  <xsl:variable name="region" select="doc('input:regionBeingProcessed')/Region/@id"/>
>                  <xsl:variable name="dataSet" select="doc('input:dataBeingProcessed')/Data/@id"/>
>                  <xsl:value-of select="fn:concat($location,'/',$name,'/',$dataSet,$region)"/>
>                </TargetURL>
>              </xsl:template>
>            </xsl:stylesheet>
>          </p:input>
>          <p:input name="data" href="#configuration"/>
>          <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>          <p:input name="dataBeingProcessed" href="#dataBeingProcessed"/>
>          <p:output name="data" id="TargetURL"/>
>        </p:processor>
>        <p:for-each href="current()" select="/Data/VCSPath" id="output2" ref="dataOutput" root="results">
>          <p:processor name="oxf:xslt">
>            <p:input name="config">
>              <xsl:stylesheet version="2.0">
>                <xsl:template match="/">
>                  <SourceURL>
>                    <xsl:variable name="locationPath" select="/Configuration/VCS/BrowserLocation/text()"/>
>                    <xsl:variable name="regionPath" select="doc('input:regionBeingProcessed')/Region/VCSPath/text()"/>
>                    <xsl:variable name="schemaPath" select="doc('input:schemaBeingProcessed')/Schema/VCSPath/text()"/>
>                    <xsl:variable name="namePath" select="doc('input:fileBeingProcessed')/VCSPath/text()"/>
>                    <xsl:value-of select="fn:string-join(($locationPath,$regionPath,$schemaPath,$namePath), '/')"/>
>                  </SourceURL>
>                </xsl:template>
>              </xsl:stylesheet>
>            </p:input>
>            <p:input name="data" href="#configuration"/>
>            <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>            <p:input name="schemaBeingProcessed" href="#schemaBeingProcessed"/>
>            <p:input name="fileBeingProcessed" href="current()"/>
>            <p:output name="data" id="SourceURL"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission">
>              <xforms:submission method="get" action="{/SourceURL/text()}" mediatype="text/xml;" serialization="none">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#SourceURL"/>
>            <p:output name="response" id="xml"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission" transform="oxf:xslt" href="#TargetURL">
>              <xforms:submission xsl:version="2.0" method="post" action="{/TargetURL/text()}" xxforms:username="admin" xxforms:password="admin"  mediatype="text/xml;">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#xml"/>
>            <p:output name="response" ref="output2"/>
>          </p:processor>
>        </p:for-each>
>      </p:for-each>
>    </p:for-each>
>  </p:for-each>
>
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws
Reply | Threaded
Open this post in threaded view
|

Re: Re: Out of memory error while attempting to load document to XML DB

Mahender Didwania
In reply to this post by Mahender Didwania
Hi,

The size of the heap is 256M (I tried with 512M too, without success).

The URL Generator does streaming, from the docs:
NOTE: The URL generator performs streaming. It generates a stream of short character SAX events. It is therefore possible to generate an "infinitely" long document with a constant amount of memory, assuming the generator is connected to other processors that do not require storing the entire stream of data in memory, for example the SQL processor (with an appropriate configuration to stream BLOBs), or the HTTP serializer.

But the URL Generator does not appear to support AVTs, e.g., the below does not work:
<p:processor name="oxf:url-generator">
     <p:input name="config" transform="oxf:xslt" href="#SourceURL">
       <config xsl:version="2.0">
        <url>{/SourceURL/text()}</url>
        <content-type>application/xml</content-type>
        <encoding>iso-8859-1</encoding>
        <force-encoding>true</force-encoding>
        <cache-control>
         <use-local-cache>false</use-local-cache>
        </cache-control>
       </config>
      </p:input>
      <p:output name="data" id="xml" />
</p:processor>

Don't know if submissions processor then can work in conjunction with URL Generator using streaming itself.

Will try with a hard-coded Source URL using URL generator and see if that leads to any different results, with 256M and 512M heap sizes; will report back.

Thanks,
Mahender.

----- Original Message -----
From: "Erik Bruchez" <[hidden email]>
To: [hidden email]
Sent: Thursday, December 16, 2010 5:36:58 AM GMT +00:00 GMT Britain, Ireland, Portugal
Subject: [ops-users] Re: Out of memory error while attempting to load document to XML DB

So what's the size of your Java heap?

I would expect a 3.5 MB XML file to translate into at least 10x that
in RAM. Multiply by a factor if the document does through "tees" in
the pipeline as the document will then be stored in memory possibly
more than once.

-Erik

On Wed, Dec 15, 2010 at 6:57 AM, Mahender Didwania
<[hidden email]> wrote:

> Hi,
>
> I fetch some documents from VCS and load them to an XML DB.  The guide code is below.  I am presently getting out of java heap space error when attempting to load only two documents in total (it succeeds with the smaller document, 631 KB in size, but fails with the larger document, which is only 3.5 MB in size.  I am using Orbeon 3.8 on apache tomcat (xampp) on a Windows XP machine with 2 gigs of memory (got 800 MB physical RAM free when Tomcat is active).  File which is loaded using url-generator processor (not shown here) and made available on #configuration input is merely 2 KB in size.
>
> Any pointers as to what I might be doing wrong?
>
>  <p:for-each href="#configuration" select="/Configuration/Regions/Region" id="output" ref="data" root="results">
>    <p:processor name="oxf:identity">
>      <p:input name="data" href="current()"/>
>      <p:output name="data" id="regionBeingProcessed"/>
>    </p:processor>
>    <p:for-each href="current()" select="/Region/Schema" id="schemaOutput" ref="output" root="resultsSchema">
>      <p:processor name="oxf:identity">
>        <p:input name="data" href="current()"/>
>        <p:output name="data" id="schemaBeingProcessed"/>
>      </p:processor>
>      <p:for-each href="current()" select="/Schema/Data" id="dataOutput" ref="schemaOutput" root="resultsData">
>        <p:processor name="oxf:identity">
>          <p:input name="data" href="current()"/>
>          <p:output name="data" id="dataBeingProcessed"/>
>        </p:processor>
>        <p:processor name="oxf:xslt">
>          <p:input name="config">
>            <xsl:stylesheet version="2.0">
>              <xsl:template match="/">
>                <TargetURL>
>                  <xsl:variable name="location" select="/Configuration/XMLDB/Location/text()"/>
>                  <xsl:variable name="name" select="/Configuration/XMLDB/Name/text()"/>
>                  <xsl:variable name="region" select="doc('input:regionBeingProcessed')/Region/@id"/>
>                  <xsl:variable name="dataSet" select="doc('input:dataBeingProcessed')/Data/@id"/>
>                  <xsl:value-of select="fn:concat($location,'/',$name,'/',$dataSet,$region)"/>
>                </TargetURL>
>              </xsl:template>
>            </xsl:stylesheet>
>          </p:input>
>          <p:input name="data" href="#configuration"/>
>          <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>          <p:input name="dataBeingProcessed" href="#dataBeingProcessed"/>
>          <p:output name="data" id="TargetURL"/>
>        </p:processor>
>        <p:for-each href="current()" select="/Data/VCSPath" id="output2" ref="dataOutput" root="results">
>          <p:processor name="oxf:xslt">
>            <p:input name="config">
>              <xsl:stylesheet version="2.0">
>                <xsl:template match="/">
>                  <SourceURL>
>                    <xsl:variable name="locationPath" select="/Configuration/VCS/BrowserLocation/text()"/>
>                    <xsl:variable name="regionPath" select="doc('input:regionBeingProcessed')/Region/VCSPath/text()"/>
>                    <xsl:variable name="schemaPath" select="doc('input:schemaBeingProcessed')/Schema/VCSPath/text()"/>
>                    <xsl:variable name="namePath" select="doc('input:fileBeingProcessed')/VCSPath/text()"/>
>                    <xsl:value-of select="fn:string-join(($locationPath,$regionPath,$schemaPath,$namePath), '/')"/>
>                  </SourceURL>
>                </xsl:template>
>              </xsl:stylesheet>
>            </p:input>
>            <p:input name="data" href="#configuration"/>
>            <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>            <p:input name="schemaBeingProcessed" href="#schemaBeingProcessed"/>
>            <p:input name="fileBeingProcessed" href="current()"/>
>            <p:output name="data" id="SourceURL"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission">
>              <xforms:submission method="get" action="{/SourceURL/text()}" mediatype="text/xml;" serialization="none">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#SourceURL"/>
>            <p:output name="response" id="xml"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission" transform="oxf:xslt" href="#TargetURL">
>              <xforms:submission xsl:version="2.0" method="post" action="{/TargetURL/text()}" xxforms:username="admin" xxforms:password="admin"  mediatype="text/xml;">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#xml"/>
>            <p:output name="response" ref="output2"/>
>          </p:processor>
>        </p:for-each>
>      </p:for-each>
>    </p:for-each>
>  </p:for-each>
>
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>

--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws
Reply | Threaded
Open this post in threaded view
|

Re: Re: Re: Out of memory error while attempting to load document to XML DB

Mahender Didwania
Hi,

Tried using URL Generator as well, but still out of memory (upto 512MB).

Perhaps need to dig deeper into submissions processor config...

Have switched to a machine with more RAM for the time being (max. 1GB given to tomcat, with permsize max at 512MB).

Regards,
Mahender

----- Original Message -----
From: "Mahender Didwania" <[hidden email]>
To: [hidden email]
Sent: Thursday, December 16, 2010 8:39:21 AM GMT +00:00 GMT Britain, Ireland, Portugal
Subject: [ops-users] Re: Re: Out of memory error while attempting to load document to XML DB

Hi,

The size of the heap is 256M (I tried with 512M too, without success).

The URL Generator does streaming, from the docs:
NOTE: The URL generator performs streaming. It generates a stream of short character SAX events. It is therefore possible to generate an "infinitely" long document with a constant amount of memory, assuming the generator is connected to other processors that do not require storing the entire stream of data in memory, for example the SQL processor (with an appropriate configuration to stream BLOBs), or the HTTP serializer.

But the URL Generator does not appear to support AVTs, e.g., the below does not work:
<p:processor name="oxf:url-generator">
     <p:input name="config" transform="oxf:xslt" href="#SourceURL">
       <config xsl:version="2.0">
        <url>{/SourceURL/text()}</url>
        <content-type>application/xml</content-type>
        <encoding>iso-8859-1</encoding>
        <force-encoding>true</force-encoding>
        <cache-control>
         <use-local-cache>false</use-local-cache>
        </cache-control>
       </config>
      </p:input>
      <p:output name="data" id="xml" />
</p:processor>

Don't know if submissions processor then can work in conjunction with URL Generator using streaming itself.

Will try with a hard-coded Source URL using URL generator and see if that leads to any different results, with 256M and 512M heap sizes; will report back.

Thanks,
Mahender.

----- Original Message -----
From: "Erik Bruchez" <[hidden email]>
To: [hidden email]
Sent: Thursday, December 16, 2010 5:36:58 AM GMT +00:00 GMT Britain, Ireland, Portugal
Subject: [ops-users] Re: Out of memory error while attempting to load document to XML DB

So what's the size of your Java heap?

I would expect a 3.5 MB XML file to translate into at least 10x that
in RAM. Multiply by a factor if the document does through "tees" in
the pipeline as the document will then be stored in memory possibly
more than once.

-Erik

On Wed, Dec 15, 2010 at 6:57 AM, Mahender Didwania
<[hidden email]> wrote:

> Hi,
>
> I fetch some documents from VCS and load them to an XML DB.  The guide code is below.  I am presently getting out of java heap space error when attempting to load only two documents in total (it succeeds with the smaller document, 631 KB in size, but fails with the larger document, which is only 3.5 MB in size.  I am using Orbeon 3.8 on apache tomcat (xampp) on a Windows XP machine with 2 gigs of memory (got 800 MB physical RAM free when Tomcat is active).  File which is loaded using url-generator processor (not shown here) and made available on #configuration input is merely 2 KB in size.
>
> Any pointers as to what I might be doing wrong?
>
>  <p:for-each href="#configuration" select="/Configuration/Regions/Region" id="output" ref="data" root="results">
>    <p:processor name="oxf:identity">
>      <p:input name="data" href="current()"/>
>      <p:output name="data" id="regionBeingProcessed"/>
>    </p:processor>
>    <p:for-each href="current()" select="/Region/Schema" id="schemaOutput" ref="output" root="resultsSchema">
>      <p:processor name="oxf:identity">
>        <p:input name="data" href="current()"/>
>        <p:output name="data" id="schemaBeingProcessed"/>
>      </p:processor>
>      <p:for-each href="current()" select="/Schema/Data" id="dataOutput" ref="schemaOutput" root="resultsData">
>        <p:processor name="oxf:identity">
>          <p:input name="data" href="current()"/>
>          <p:output name="data" id="dataBeingProcessed"/>
>        </p:processor>
>        <p:processor name="oxf:xslt">
>          <p:input name="config">
>            <xsl:stylesheet version="2.0">
>              <xsl:template match="/">
>                <TargetURL>
>                  <xsl:variable name="location" select="/Configuration/XMLDB/Location/text()"/>
>                  <xsl:variable name="name" select="/Configuration/XMLDB/Name/text()"/>
>                  <xsl:variable name="region" select="doc('input:regionBeingProcessed')/Region/@id"/>
>                  <xsl:variable name="dataSet" select="doc('input:dataBeingProcessed')/Data/@id"/>
>                  <xsl:value-of select="fn:concat($location,'/',$name,'/',$dataSet,$region)"/>
>                </TargetURL>
>              </xsl:template>
>            </xsl:stylesheet>
>          </p:input>
>          <p:input name="data" href="#configuration"/>
>          <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>          <p:input name="dataBeingProcessed" href="#dataBeingProcessed"/>
>          <p:output name="data" id="TargetURL"/>
>        </p:processor>
>        <p:for-each href="current()" select="/Data/VCSPath" id="output2" ref="dataOutput" root="results">
>          <p:processor name="oxf:xslt">
>            <p:input name="config">
>              <xsl:stylesheet version="2.0">
>                <xsl:template match="/">
>                  <SourceURL>
>                    <xsl:variable name="locationPath" select="/Configuration/VCS/BrowserLocation/text()"/>
>                    <xsl:variable name="regionPath" select="doc('input:regionBeingProcessed')/Region/VCSPath/text()"/>
>                    <xsl:variable name="schemaPath" select="doc('input:schemaBeingProcessed')/Schema/VCSPath/text()"/>
>                    <xsl:variable name="namePath" select="doc('input:fileBeingProcessed')/VCSPath/text()"/>
>                    <xsl:value-of select="fn:string-join(($locationPath,$regionPath,$schemaPath,$namePath), '/')"/>
>                  </SourceURL>
>                </xsl:template>
>              </xsl:stylesheet>
>            </p:input>
>            <p:input name="data" href="#configuration"/>
>            <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>            <p:input name="schemaBeingProcessed" href="#schemaBeingProcessed"/>
>            <p:input name="fileBeingProcessed" href="current()"/>
>            <p:output name="data" id="SourceURL"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission">
>              <xforms:submission method="get" action="{/SourceURL/text()}" mediatype="text/xml;" serialization="none">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#SourceURL"/>
>            <p:output name="response" id="xml"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission" transform="oxf:xslt" href="#TargetURL">
>              <xforms:submission xsl:version="2.0" method="post" action="{/TargetURL/text()}" xxforms:username="admin" xxforms:password="admin"  mediatype="text/xml;">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#xml"/>
>            <p:output name="response" ref="output2"/>
>          </p:processor>
>        </p:for-each>
>      </p:for-each>
>    </p:for-each>
>  </p:for-each>
>
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>

--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws
Reply | Threaded
Open this post in threaded view
|

Re: Re: Re: Re: Out of memory error while attempting to load document to XML DB

Mahender Didwania
Hi,

How do I/Is there a way to configure the submissions processor to not try parsing (and creating the DOM tree) for the request and response instances immediately (meaning thereby, how to configure the DOM creation to be lazy, i.e., only if one tries to perform a query against the instance; this will help my case because then hopefully 10x memory (compared to doc size) won't be consumed because I do not perform any query against the instance anyway - I just post it, as it is, to a URL)?

Regards,
Mahender.
----- Original Message -----
From: "Mahender Didwania" <[hidden email]>
To: [hidden email]
Sent: Thursday, December 16, 2010 10:40:01 AM GMT +00:00 GMT Britain, Ireland, Portugal
Subject: [ops-users] Re: Re: Re: Out of memory error while attempting to load document to XML DB

Hi,

Tried using URL Generator as well, but still out of memory (upto 512MB).

Perhaps need to dig deeper into submissions processor config...

Have switched to a machine with more RAM for the time being (max. 1GB given to tomcat, with permsize max at 512MB).

Regards,
Mahender

----- Original Message -----
From: "Mahender Didwania" <[hidden email]>
To: [hidden email]
Sent: Thursday, December 16, 2010 8:39:21 AM GMT +00:00 GMT Britain, Ireland, Portugal
Subject: [ops-users] Re: Re: Out of memory error while attempting to load document to XML DB

Hi,

The size of the heap is 256M (I tried with 512M too, without success).

The URL Generator does streaming, from the docs:
NOTE: The URL generator performs streaming. It generates a stream of short character SAX events. It is therefore possible to generate an "infinitely" long document with a constant amount of memory, assuming the generator is connected to other processors that do not require storing the entire stream of data in memory, for example the SQL processor (with an appropriate configuration to stream BLOBs), or the HTTP serializer.

But the URL Generator does not appear to support AVTs, e.g., the below does not work:
<p:processor name="oxf:url-generator">
     <p:input name="config" transform="oxf:xslt" href="#SourceURL">
       <config xsl:version="2.0">
        <url>{/SourceURL/text()}</url>
        <content-type>application/xml</content-type>
        <encoding>iso-8859-1</encoding>
        <force-encoding>true</force-encoding>
        <cache-control>
         <use-local-cache>false</use-local-cache>
        </cache-control>
       </config>
      </p:input>
      <p:output name="data" id="xml" />
</p:processor>

Don't know if submissions processor then can work in conjunction with URL Generator using streaming itself.

Will try with a hard-coded Source URL using URL generator and see if that leads to any different results, with 256M and 512M heap sizes; will report back.

Thanks,
Mahender.

----- Original Message -----
From: "Erik Bruchez" <[hidden email]>
To: [hidden email]
Sent: Thursday, December 16, 2010 5:36:58 AM GMT +00:00 GMT Britain, Ireland, Portugal
Subject: [ops-users] Re: Out of memory error while attempting to load document to XML DB

So what's the size of your Java heap?

I would expect a 3.5 MB XML file to translate into at least 10x that
in RAM. Multiply by a factor if the document does through "tees" in
the pipeline as the document will then be stored in memory possibly
more than once.

-Erik

On Wed, Dec 15, 2010 at 6:57 AM, Mahender Didwania
<[hidden email]> wrote:

> Hi,
>
> I fetch some documents from VCS and load them to an XML DB.  The guide code is below.  I am presently getting out of java heap space error when attempting to load only two documents in total (it succeeds with the smaller document, 631 KB in size, but fails with the larger document, which is only 3.5 MB in size.  I am using Orbeon 3.8 on apache tomcat (xampp) on a Windows XP machine with 2 gigs of memory (got 800 MB physical RAM free when Tomcat is active).  File which is loaded using url-generator processor (not shown here) and made available on #configuration input is merely 2 KB in size.
>
> Any pointers as to what I might be doing wrong?
>
>  <p:for-each href="#configuration" select="/Configuration/Regions/Region" id="output" ref="data" root="results">
>    <p:processor name="oxf:identity">
>      <p:input name="data" href="current()"/>
>      <p:output name="data" id="regionBeingProcessed"/>
>    </p:processor>
>    <p:for-each href="current()" select="/Region/Schema" id="schemaOutput" ref="output" root="resultsSchema">
>      <p:processor name="oxf:identity">
>        <p:input name="data" href="current()"/>
>        <p:output name="data" id="schemaBeingProcessed"/>
>      </p:processor>
>      <p:for-each href="current()" select="/Schema/Data" id="dataOutput" ref="schemaOutput" root="resultsData">
>        <p:processor name="oxf:identity">
>          <p:input name="data" href="current()"/>
>          <p:output name="data" id="dataBeingProcessed"/>
>        </p:processor>
>        <p:processor name="oxf:xslt">
>          <p:input name="config">
>            <xsl:stylesheet version="2.0">
>              <xsl:template match="/">
>                <TargetURL>
>                  <xsl:variable name="location" select="/Configuration/XMLDB/Location/text()"/>
>                  <xsl:variable name="name" select="/Configuration/XMLDB/Name/text()"/>
>                  <xsl:variable name="region" select="doc('input:regionBeingProcessed')/Region/@id"/>
>                  <xsl:variable name="dataSet" select="doc('input:dataBeingProcessed')/Data/@id"/>
>                  <xsl:value-of select="fn:concat($location,'/',$name,'/',$dataSet,$region)"/>
>                </TargetURL>
>              </xsl:template>
>            </xsl:stylesheet>
>          </p:input>
>          <p:input name="data" href="#configuration"/>
>          <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>          <p:input name="dataBeingProcessed" href="#dataBeingProcessed"/>
>          <p:output name="data" id="TargetURL"/>
>        </p:processor>
>        <p:for-each href="current()" select="/Data/VCSPath" id="output2" ref="dataOutput" root="results">
>          <p:processor name="oxf:xslt">
>            <p:input name="config">
>              <xsl:stylesheet version="2.0">
>                <xsl:template match="/">
>                  <SourceURL>
>                    <xsl:variable name="locationPath" select="/Configuration/VCS/BrowserLocation/text()"/>
>                    <xsl:variable name="regionPath" select="doc('input:regionBeingProcessed')/Region/VCSPath/text()"/>
>                    <xsl:variable name="schemaPath" select="doc('input:schemaBeingProcessed')/Schema/VCSPath/text()"/>
>                    <xsl:variable name="namePath" select="doc('input:fileBeingProcessed')/VCSPath/text()"/>
>                    <xsl:value-of select="fn:string-join(($locationPath,$regionPath,$schemaPath,$namePath), '/')"/>
>                  </SourceURL>
>                </xsl:template>
>              </xsl:stylesheet>
>            </p:input>
>            <p:input name="data" href="#configuration"/>
>            <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>            <p:input name="schemaBeingProcessed" href="#schemaBeingProcessed"/>
>            <p:input name="fileBeingProcessed" href="current()"/>
>            <p:output name="data" id="SourceURL"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission">
>              <xforms:submission method="get" action="{/SourceURL/text()}" mediatype="text/xml;" serialization="none">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#SourceURL"/>
>            <p:output name="response" id="xml"/>
>          </p:processor>
>          <p:processor name="oxf:xforms-submission">
>            <p:input name="submission" transform="oxf:xslt" href="#TargetURL">
>              <xforms:submission xsl:version="2.0" method="post" action="{/TargetURL/text()}" xxforms:username="admin" xxforms:password="admin"  mediatype="text/xml;">
>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>                  <xforms:output value="event('error-type')"/>
>                </xforms:message>
>              </xforms:submission>
>            </p:input>
>            <p:input name="request" href="#xml"/>
>            <p:output name="response" ref="output2"/>
>          </p:processor>
>        </p:for-each>
>      </p:for-each>
>    </p:for-each>
>  </p:for-each>
>
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>

--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws
Reply | Threaded
Open this post in threaded view
|

Re: Re: Re: Re: Re: Out of memory error while attempting to load document to XML DB

Erik Bruchez
Administrator
In XPL, if you don't read a processor's outputs, it won't read its
inputs either.

So with oxf:xforms-submission, the only way to prevent it from reading
its input is to not read its output. I am not sure if that makes sense
or if that helps in your case!

-Erik

On Thu, Dec 16, 2010 at 5:29 AM, Mahender Didwania
<[hidden email]> wrote:

> Hi,
>
> How do I/Is there a way to configure the submissions processor to not try parsing (and creating the DOM tree) for the request and response instances immediately (meaning thereby, how to configure the DOM creation to be lazy, i.e., only if one tries to perform a query against the instance; this will help my case because then hopefully 10x memory (compared to doc size) won't be consumed because I do not perform any query against the instance anyway - I just post it, as it is, to a URL)?
>
> Regards,
> Mahender.
> ----- Original Message -----
> From: "Mahender Didwania" <[hidden email]>
> To: [hidden email]
> Sent: Thursday, December 16, 2010 10:40:01 AM GMT +00:00 GMT Britain, Ireland, Portugal
> Subject: [ops-users] Re: Re: Re: Out of memory error while attempting to load document to XML DB
>
> Hi,
>
> Tried using URL Generator as well, but still out of memory (upto 512MB).
>
> Perhaps need to dig deeper into submissions processor config...
>
> Have switched to a machine with more RAM for the time being (max. 1GB given to tomcat, with permsize max at 512MB).
>
> Regards,
> Mahender
>
> ----- Original Message -----
> From: "Mahender Didwania" <[hidden email]>
> To: [hidden email]
> Sent: Thursday, December 16, 2010 8:39:21 AM GMT +00:00 GMT Britain, Ireland, Portugal
> Subject: [ops-users] Re: Re: Out of memory error while attempting to load document to XML DB
>
> Hi,
>
> The size of the heap is 256M (I tried with 512M too, without success).
>
> The URL Generator does streaming, from the docs:
> NOTE: The URL generator performs streaming. It generates a stream of short character SAX events. It is therefore possible to generate an "infinitely" long document with a constant amount of memory, assuming the generator is connected to other processors that do not require storing the entire stream of data in memory, for example the SQL processor (with an appropriate configuration to stream BLOBs), or the HTTP serializer.
>
> But the URL Generator does not appear to support AVTs, e.g., the below does not work:
> <p:processor name="oxf:url-generator">
>     <p:input name="config" transform="oxf:xslt" href="#SourceURL">
>       <config xsl:version="2.0">
>        <url>{/SourceURL/text()}</url>
>        <content-type>application/xml</content-type>
>        <encoding>iso-8859-1</encoding>
>        <force-encoding>true</force-encoding>
>        <cache-control>
>         <use-local-cache>false</use-local-cache>
>        </cache-control>
>       </config>
>      </p:input>
>      <p:output name="data" id="xml" />
> </p:processor>
>
> Don't know if submissions processor then can work in conjunction with URL Generator using streaming itself.
>
> Will try with a hard-coded Source URL using URL generator and see if that leads to any different results, with 256M and 512M heap sizes; will report back.
>
> Thanks,
> Mahender.
>
> ----- Original Message -----
> From: "Erik Bruchez" <[hidden email]>
> To: [hidden email]
> Sent: Thursday, December 16, 2010 5:36:58 AM GMT +00:00 GMT Britain, Ireland, Portugal
> Subject: [ops-users] Re: Out of memory error while attempting to load document to XML DB
>
> So what's the size of your Java heap?
>
> I would expect a 3.5 MB XML file to translate into at least 10x that
> in RAM. Multiply by a factor if the document does through "tees" in
> the pipeline as the document will then be stored in memory possibly
> more than once.
>
> -Erik
>
> On Wed, Dec 15, 2010 at 6:57 AM, Mahender Didwania
> <[hidden email]> wrote:
>> Hi,
>>
>> I fetch some documents from VCS and load them to an XML DB.  The guide code is below.  I am presently getting out of java heap space error when attempting to load only two documents in total (it succeeds with the smaller document, 631 KB in size, but fails with the larger document, which is only 3.5 MB in size.  I am using Orbeon 3.8 on apache tomcat (xampp) on a Windows XP machine with 2 gigs of memory (got 800 MB physical RAM free when Tomcat is active).  File which is loaded using url-generator processor (not shown here) and made available on #configuration input is merely 2 KB in size.
>>
>> Any pointers as to what I might be doing wrong?
>>
>>  <p:for-each href="#configuration" select="/Configuration/Regions/Region" id="output" ref="data" root="results">
>>    <p:processor name="oxf:identity">
>>      <p:input name="data" href="current()"/>
>>      <p:output name="data" id="regionBeingProcessed"/>
>>    </p:processor>
>>    <p:for-each href="current()" select="/Region/Schema" id="schemaOutput" ref="output" root="resultsSchema">
>>      <p:processor name="oxf:identity">
>>        <p:input name="data" href="current()"/>
>>        <p:output name="data" id="schemaBeingProcessed"/>
>>      </p:processor>
>>      <p:for-each href="current()" select="/Schema/Data" id="dataOutput" ref="schemaOutput" root="resultsData">
>>        <p:processor name="oxf:identity">
>>          <p:input name="data" href="current()"/>
>>          <p:output name="data" id="dataBeingProcessed"/>
>>        </p:processor>
>>        <p:processor name="oxf:xslt">
>>          <p:input name="config">
>>            <xsl:stylesheet version="2.0">
>>              <xsl:template match="/">
>>                <TargetURL>
>>                  <xsl:variable name="location" select="/Configuration/XMLDB/Location/text()"/>
>>                  <xsl:variable name="name" select="/Configuration/XMLDB/Name/text()"/>
>>                  <xsl:variable name="region" select="doc('input:regionBeingProcessed')/Region/@id"/>
>>                  <xsl:variable name="dataSet" select="doc('input:dataBeingProcessed')/Data/@id"/>
>>                  <xsl:value-of select="fn:concat($location,'/',$name,'/',$dataSet,$region)"/>
>>                </TargetURL>
>>              </xsl:template>
>>            </xsl:stylesheet>
>>          </p:input>
>>          <p:input name="data" href="#configuration"/>
>>          <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>>          <p:input name="dataBeingProcessed" href="#dataBeingProcessed"/>
>>          <p:output name="data" id="TargetURL"/>
>>        </p:processor>
>>        <p:for-each href="current()" select="/Data/VCSPath" id="output2" ref="dataOutput" root="results">
>>          <p:processor name="oxf:xslt">
>>            <p:input name="config">
>>              <xsl:stylesheet version="2.0">
>>                <xsl:template match="/">
>>                  <SourceURL>
>>                    <xsl:variable name="locationPath" select="/Configuration/VCS/BrowserLocation/text()"/>
>>                    <xsl:variable name="regionPath" select="doc('input:regionBeingProcessed')/Region/VCSPath/text()"/>
>>                    <xsl:variable name="schemaPath" select="doc('input:schemaBeingProcessed')/Schema/VCSPath/text()"/>
>>                    <xsl:variable name="namePath" select="doc('input:fileBeingProcessed')/VCSPath/text()"/>
>>                    <xsl:value-of select="fn:string-join(($locationPath,$regionPath,$schemaPath,$namePath), '/')"/>
>>                  </SourceURL>
>>                </xsl:template>
>>              </xsl:stylesheet>
>>            </p:input>
>>            <p:input name="data" href="#configuration"/>
>>            <p:input name="regionBeingProcessed" href="#regionBeingProcessed"/>
>>            <p:input name="schemaBeingProcessed" href="#schemaBeingProcessed"/>
>>            <p:input name="fileBeingProcessed" href="current()"/>
>>            <p:output name="data" id="SourceURL"/>
>>          </p:processor>
>>          <p:processor name="oxf:xforms-submission">
>>            <p:input name="submission">
>>              <xforms:submission method="get" action="{/SourceURL/text()}" mediatype="text/xml;" serialization="none">
>>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>>                  <xforms:output value="event('error-type')"/>
>>                </xforms:message>
>>              </xforms:submission>
>>            </p:input>
>>            <p:input name="request" href="#SourceURL"/>
>>            <p:output name="response" id="xml"/>
>>          </p:processor>
>>          <p:processor name="oxf:xforms-submission">
>>            <p:input name="submission" transform="oxf:xslt" href="#TargetURL">
>>              <xforms:submission xsl:version="2.0" method="post" action="{/TargetURL/text()}" xxforms:username="admin" xxforms:password="admin"  mediatype="text/xml;">
>>                <xforms:message ev:event="xforms-submit-error" level="modal">A submission error occurred:
>>                  <xforms:output value="event('error-type')"/>
>>                </xforms:message>
>>              </xforms:submission>
>>            </p:input>
>>            <p:input name="request" href="#xml"/>
>>            <p:output name="response" ref="output2"/>
>>          </p:processor>
>>        </p:for-each>
>>      </p:for-each>
>>    </p:for-each>
>>  </p:for-each>
>>
>>
>>
>> --
>> You receive this message as a subscriber of the [hidden email] mailing list.
>> To unsubscribe: mailto:[hidden email]
>> For general help: mailto:[hidden email]?subject=help
>> OW2 mailing lists service home page: http://www.ow2.org/wws
>>
>>
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>
> --
> You receive this message as a subscriber of the [hidden email] mailing list.
> To unsubscribe: mailto:[hidden email]
> For general help: mailto:[hidden email]?subject=help
> OW2 mailing lists service home page: http://www.ow2.org/wws
>
>


--
You receive this message as a subscriber of the [hidden email] mailing list.
To unsubscribe: mailto:[hidden email]
For general help: mailto:[hidden email]?subject=help
OW2 mailing lists service home page: http://www.ow2.org/wws