Home » SQL & PL/SQL » SQL & PL/SQL » procedure taking time (oracle,9.2.0.8,window)
procedure taking time [message #447923] |
Thu, 18 March 2010 08:29 |
|
Hi All,
Mentioned below procedure is taking more then 20 hrs. Could you please advice what could be the problem in this procedure
CREATE OR REPLACE PROCEDURE geidm.longvarchar_to_blob (
oldtablename IN VARCHAR,
newtablename IN VARCHAR,
idcolumnname IN VARCHAR,
oldlobcolumnname IN VARCHAR,
newlobcolumnname IN VARCHAR
)
AUTHID CURRENT_USER
IS
TYPE mycurtype IS REF CURSOR;
my_cursor mycurtype;
clob_loc NCLOB;
blob_loc BLOB;
charbuf NVARCHAR2 (8000);
rawbuf RAW (32000);
amount NUMBER;
offset NUMBER;
LENGTH NUMBER;
ID VARCHAR (50);
counter NUMBER := 0;
BEGIN
EXECUTE IMMEDIATE 'CREATE TABLE geidm.tempnclob (tempnclob NCLOB)';
OPEN my_cursor FOR 'select ' || idcolumnname || ' FROM ' || newtablename;
LOOP
FETCH my_cursor
INTO ID;
EXIT WHEN my_cursor%NOTFOUND;
--First convert the LONG VARCHAR to an NCLOB
EXECUTE IMMEDIATE 'INSERT INTO geidm.tempnclob select TO_LOB('
|| oldlobcolumnname
|| ') FROM '
|| oldtablename
|| ' WHERE '
|| idcolumnname
|| ' = :1'
USING ID;
--Get the clob_loc
EXECUTE IMMEDIATE 'select tempnclob FROM geidm.tempnclob'
INTO clob_loc;
--Set the empty_blob
EXECUTE IMMEDIATE 'update '
|| newtablename
|| ' SET '
|| newlobcolumnname
|| ' = empty_blob() '
|| ' WHERE '
|| idcolumnname
|| ' = :1'
USING ID;
--Get the blob_loc
EXECUTE IMMEDIATE 'select '
|| newlobcolumnname
|| ' FROM '
|| newtablename
|| ' WHERE '
|| idcolumnname
|| ' = :1 FOR UPDATE'
INTO blob_loc
USING ID;
--convert the nclob to a blob
DBMS_LOB.OPEN (blob_loc, DBMS_LOB.lob_readwrite);
DBMS_LOB.OPEN (clob_loc, DBMS_LOB.lob_readonly);
offset := 1;
LENGTH := DBMS_LOB.getlength (clob_loc);
WHILE (offset <= LENGTH)
LOOP
amount := 8000;
DBMS_LOB.READ (clob_loc, amount, offset, charbuf);
rawbuf := UTL_RAW.cast_to_raw (charbuf);
DBMS_LOB.writeappend (blob_loc, UTL_RAW.LENGTH (rawbuf), rawbuf);
offset := offset + amount;
END LOOP;
DBMS_LOB.CLOSE (blob_loc);
DBMS_LOB.CLOSE (clob_loc);
--delete the temporary nclob
EXECUTE IMMEDIATE 'delete from geidm.tempnclob';
--commit every 100 iterations to avoid exhausting rollback segments
counter := counter + 1;
IF (MOD (counter, 100) = 0)
THEN
COMMIT;
END IF;
END LOOP;
CLOSE my_cursor;
EXECUTE IMMEDIATE 'DROP TABLE geidm.tempnclob';
COMMIT;
END;
/
Thanks,
Sagar
|
|
|
|
Re: procedure taking time [message #447932 is a reply to message #447923] |
Thu, 18 March 2010 09:51 |
JRowbottom
Messages: 5933 Registered: June 2006 Location: Sunny North Yorkshire, ho...
|
Senior Member |
|
|
It's slow basically because you're looping through a table, and doing 5 dynamic SQL statements per row.
Do you really have so many tables to run this conversion on that you couldn't just create a procedure for each? Dynamic SQL makes the whole thing slower.
It looks like you're trying to convert a LONG column to a BLOB, but that doesn't make a lot of sense, as LONGs hold character data, and should get converted to CLOBS - can you provide a little info about what you're trying to do.
This may work (I havent got a 9.2 db to try it out on)
...
BEGIN
EXECUTE IMMEDIATE 'CREATE TABLE geidm.tempnclob (id varchar2(50), tempblob BLOB)';
EXECUTE IMMEDIATE 'INSERT INTO geidm.tempnclob '||
' select '||idcolumnname||', TO_LOB('
|| oldlobcolumnname
|| ') FROM '
|| oldtablename;
execute immediate
'UPDATE '||newtablename||' n set '||
newcolumnname||' = (select '||oldcolumnname||
'from '||oldtablename||' o '||
'where n.'||idcolumnname||'= o.'||idcolumnname||')';
EXECUTE IMMEDIATE 'DROP TABLE geidm.tempnclob';
end;
Failing that, I might be tempted to try:CREATE OR REPLACE FUNCTION nclob_to_blob (p_nclob in nclob) return blob as
v_return blob := empty_blob();
v_length pls_integer;
v_offset pls_integer := 0;
v_amount pls_integer;
v_raw raw(32000);
v_vc varchar2(32000);
begin
v_LENGTH := DBMS_LOB.getlength(p_nclob);
WHILE (v_offset <= LENGTH) LOOP
v_amount := 8000;
DBMS_LOB.READ (p_nclob, v_amount, v_offset, v_vc2);
rawbuf := UTL_RAW.cast_to_raw (v_vc2);
DBMS_LOB.writeappend (v_return, UTL_RAW.LENGTH (v_raw), v_raw);
v_offset := v_offset + v_amount;
END LOOP;
return v_return;
end;
Then I'd create and populate a table as you do originally, and then run a single update to convert the NCLOB to BLOB and populate the new table.
|
|
|
|
Re: procedure taking time [message #448136 is a reply to message #447986] |
Fri, 19 March 2010 13:17 |
knw15pwr
Messages: 134 Registered: March 2010
|
Senior Member |
|
|
I tried something like this.....
CREATE OR REPLACE PROCEDURE longvarchar_to_blob( oldTableName IN VARCHAR,
newTableName IN VARCHAR,
idColumnName IN VARCHAR,
oldLobColumnName IN VARCHAR,
newLobColumnName IN VARCHAR)
AUTHID CURRENT_USER
IS
/*TYPE MyCurType IS REF CURSOR;
my_cursor MyCurType;
clob_loc NCLOB;
blob_loc BLOB;
charbuf NVARCHAR2(8000);
rawbuf RAW(32000);
amount number;
offset number;
length number;
id VARCHAR(50);*/
counter number := 0;
BEGIN
EXECUTE IMMEDIATE 'CREATE TABLE tempnclob (id varchar2(50), tempnclob NCLOB)';
EXECUTE IMMEDIATE 'INSERT INTO tempnclob '||
' select '||idcolumnname||', TO_LOB('
|| oldlobcolumnname
|| ') FROM '
|| oldtablename;
commit;
execute immediate
'UPDATE '||newtablename||' n set '||
newLobColumnName||' = (select empty_blob() from tempnclob o '|| 'where n.'||idcolumnname||'= o.'||idcolumnname||')';
execute immediate
'UPDATE '||newtablename||' n set '||
newLobColumnName||' = (select nclob_to_blob(tempnclob) from tempnclob o '|| 'where n.'||idcolumnname||'= o.'||idcolumnname||')';
EXECUTE IMMEDIATE 'DROP TABLE tempnclob ';
commit;
end;
/
CREATE OR REPLACE FUNCTION nclob_to_blob (p_nclob in nclob) return blob as
v_return blob := empty_blob();
v_length pls_integer;
v_offset pls_integer := 1;
v_amount pls_integer;
v_raw raw(32000);
v_vc nvarchar2(32000);
begin
v_LENGTH := DBMS_LOB.getlength(p_nclob);
DBMS_LOB.createtemporary(v_return,true);
WHILE (v_offset <= v_LENGTH) LOOP
v_amount := 8000;
DBMS_LOB.READ (p_nclob, v_amount, v_offset, v_vc);
v_raw := UTL_RAW.cast_to_raw (v_vc);
DBMS_LOB.writeappend (v_return, UTL_RAW.LENGTH (v_raw), v_raw);
v_offset := v_offset + v_amount;
END LOOP;
return v_return;
end;
/
1. It is not inserting any data in the new table though the procedure executes successfully without any errors.
2. I suppose the update will be calling the function for each record and this may affect the performance.
Is there a better way to do this ?
My requirement would be - Old table has data in long. Convert this data and insert in New table which has datatype as CLOB.
Both tables share the common ID. Basically long varchar to CLOB conversion.
Thanks,
Ritesh
|
|
|
|
|
|
Re: procedure taking time [message #448173 is a reply to message #448168] |
Sat, 20 March 2010 01:19 |
knw15pwr
Messages: 134 Registered: March 2010
|
Senior Member |
|
|
Sorry about that. Attached all the updated codes in below post.
Test data setup :
--create source table
CREATE TABLE OBJECT
( ID VARCHAR2(50 BYTE) NOT NULL, XML LONG NOT NULL);
--create target table
CREATE TABLE task
(ID VARCHAR2(50 BYTE) NOT NULL, XML BLOB);
--create temp table
CREATE TABLE tempnclob (id varchar2(50), tempnclob NCLOB);
INSERT INTO OBJECT values (1,'zzzzzz');
INSERT INTO OBJECT values (2,'ksiutgoihoasr');
INSERT INTO OBJECT values (3,'ksiutgoihoasr');
INSERT INTO OBJECT values (4,'jkadgfuqhgoihoigfhiodghoidhgoahgd');
--new table task should have the id column matching with old
INSERT INTO task(id) (select id from object);
commit;
Procedure and Function :
CREATE OR REPLACE PROCEDURE longvarchar_to_blob( oldTableName IN VARCHAR,
newTableName IN VARCHAR,
idColumnName IN VARCHAR,
oldLobColumnName IN VARCHAR,
newLobColumnName IN VARCHAR)
AUTHID CURRENT_USER
IS
BEGIN
--Insert all the long values into the temp table
EXECUTE IMMEDIATE 'INSERT INTO tempnclob '||
' select '||idcolumnname||', TO_LOB('
|| oldlobcolumnname
|| ') FROM '
|| oldtablename;
commit;
--Update the CLOB column in the new table with an empty_blob()
execute immediate
'UPDATE '||newtablename||' n set '||
newLobColumnName||' = (select empty_blob() from tempnclob o '|| 'where n.'||idcolumnname||'= o.'||idcolumnname||')';
--Update the new table using data from the temp table by calling function nclob_to_blob
execute immediate
'UPDATE '||newtablename||' n set '||
newLobColumnName||' = (select nclob_to_blob(tempnclob) from tempnclob o '|| 'where n.'||idcolumnname||'= o.'||idcolumnname||')';
commit;
end;
/
CREATE OR REPLACE FUNCTION nclob_to_blob (p_nclob in nclob) return blob as
v_return blob := empty_blob();
v_length pls_integer;
v_offset pls_integer := 1;
v_amount pls_integer;
v_raw raw(32000);
v_vc nvarchar2(32000);
begin
v_LENGTH := DBMS_LOB.getlength(p_nclob);
DBMS_LOB.createtemporary(v_return,true);
WHILE (v_offset <= v_LENGTH) LOOP
v_amount := 8000;
DBMS_LOB.READ (p_nclob, v_amount, v_offset, v_vc);
v_raw := UTL_RAW.cast_to_raw (v_vc);
DBMS_LOB.writeappend (v_return, UTL_RAW.LENGTH (v_raw), v_raw);
v_offset := v_offset + v_amount;
END LOOP;
return v_return;
end;
/
Results :
--before execution
1* select id,DBMS_LOB.getlength(xml) from task
SQL> /
ID DBMS_LOB.GETLENGTH(XML)
-------------------------------------------------- -----------------------
1
2
6
4
1 begin
2 longvarchar_to_blob('object','task','id','xml','xml');
3* end;
SQL> /
PL/SQL procedure successfully completed.
1* select id,DBMS_LOB.getlength(xml) from task
SQL> /
ID DBMS_LOB.GETLENGTH(XML)
-------------------------------------------------- -----------------------
1 12
2 26
6 26
4 66
It is working good. (I had missed synchronizing the id in both tables).
My concern : The last update statement in the procedure for newtable calls the function for each row. Considering a large dataset (say 20 to 30k records), will this statement impact the performance ?
If so, is there any alternate way to do it ?
Thanks,
Ritesh
|
|
|
Re: procedure taking time [message #448240 is a reply to message #448173] |
Mon, 22 March 2010 04:48 |
JRowbottom
Messages: 5933 Registered: June 2006 Location: Sunny North Yorkshire, ho...
|
Senior Member |
|
|
I'm pretty sure you can tweak the performance by removing the first update - the one setting the values to empty_blob(). Every row that the first statement updates will be overwritten by the second statement.
You're going to have to call the function for each row - there's no way round this.
|
|
|
|
|
Re: procedure taking time [message #448402 is a reply to message #448399] |
Tue, 23 March 2010 03:52 |
JRowbottom
Messages: 5933 Registered: June 2006 Location: Sunny North Yorkshire, ho...
|
Senior Member |
|
|
I've never heard of a requirement to initialise a Lob in a table, but I've never used the reporting tools youe using.
I'd be suprised if it was neccessary.
I'd check about this BLOB/CLOB thing if I were you - storing character data n a BLOB doesn't make any sense to me.
If you can use a CLOB column, then your code gets even simpler:CREATE OR REPLACE PROCEDURE longvarchar_to_blob( oldTableName IN VARCHAR,
newTableName IN VARCHAR,
idColumnName IN VARCHAR,
oldLobColumnName IN VARCHAR,
newLobColumnName IN VARCHAR)
AUTHID CURRENT_USER IS
BEGIN
--Insert all the long values into the temp table
EXECUTE IMMEDIATE 'INSERT INTO tempnclob '||
' select '||idcolumnname||', TO_LOB('
|| oldlobcolumnname
|| ') FROM '
|| oldtablename;
--Update the new table using data from the temp table
execute immediate
'UPDATE '||newtablename||' n set '||
newLobColumnName||' = (select tempnclob from tempnclob o '|| 'where n.'||idcolumnname||'= o.'||idcolumnname||')';
commit;
end;
/
|
|
|
Re: procedure taking time [message #448424 is a reply to message #448402] |
Tue, 23 March 2010 05:44 |
knw15pwr
Messages: 134 Registered: March 2010
|
Senior Member |
|
|
hey this works like a breeze for CLOB !
Apparently i wont even need the temporary table. I can directly update the newtable with the oldtable values using the to_lob function. Please correct me if this approach is wrong.
CREATE OR REPLACE PROCEDURE longvarchar_to_clob( oldTableName IN VARCHAR,
newTableName IN VARCHAR,
idColumnName IN VARCHAR,
oldLobColumnName IN VARCHAR,
newLobColumnName IN VARCHAR)
AUTHID CURRENT_USER IS
BEGIN
--Update the new table using data from the oldtable. Implement the function to_lob()
execute immediate
'UPDATE '||newtablename||' n set '||
newLobColumnName||' = (select TO_LOB(o.'|| oldlobcolumnname|| ') from '|| oldtablename ||' o where n.'||idcolumnname||'= o.'||idcolumnname||')';
commit;
end;
/
|
|
|
|
Re: procedure taking time [message #448437 is a reply to message #448431] |
Tue, 23 March 2010 07:00 |
knw15pwr
Messages: 134 Registered: March 2010
|
Senior Member |
|
|
It actually did!
I don't know how to post the results here as im working in Toad.
But if that is what the documents say then i'll rather keep the temporary table to be on the safe side.
|
|
|
Re: procedure taking time [message #448439 is a reply to message #448437] |
Tue, 23 March 2010 07:07 |
JRowbottom
Messages: 5933 Registered: June 2006 Location: Sunny North Yorkshire, ho...
|
Senior Member |
|
|
Interesting - I just ran some tests, and TO_LOB fails if you use it in a straight UPDATE statment, but works if you use it in a sub-query in an UPDATE statement.
|
|
|
Goto Forum:
Current Time: Wed Dec 04 19:36:30 CST 2024
|