I am using the following query.
But, it just transfers the top 1000 rows, thats it. Even though I have more rows.
If I remove the where not exists clause, I get full data. Can you let me know where am I wrong ?
DECLARE #BatchSize INT = 1000
DECLARE #Counter INT = 0
DECLARE #TableCount INT = 0
set #TableCount = (select count(*) from Table2)
while #Counter < (#TableCount/#BatchSize+1)
BEGIN
INSERT INTO Table1
SELECT * FROM Table2 MH
inner join Table3 M
on MH.Mid = M.Mid
WHERE NOT EXISTS (
SELECT * FROM Table1
where MH.otherid = M.otherid
)
order by id OFFSET (#BatchSize * #Counter)ROWS FETCH NEXT #Batchsize ROWS ONLY;
SET #Counter=#Counter+1
END
Why is it just inserting top 1000 rows ?
WHERE NOT EXISTS (
SELECT 1 FROM Table1
)
will only evaluate to true for the first batch of inserts. After that, there are records in the target table, the WHERE clause evaluates to false, so no further inserts happen.
How many rows are in Table2? Your second loop is going to produce the WHILE condition of (1 < (TableCount / 1001) if you're Table Count is more than 1001 rows, it will jump out after the first loop.
DECLARE #BatchSize INT = 1000
DECLARE #Counter INT = 0
--DECLARE #TableCount INT = 0
--set #TableCount = (select count(*) from Table2)
declare #rows int = 1
while #rows>0--#Counter < (#TableCount/#BatchSize+1)
BEGIN
INSERT INTO Table1
SELECT * FROM Table2 MH
inner join Table3 M
on MH.Mid = M.Mid
--if you want non-existing
left join table1 t on t.field = M.field
where t.field is null
-- end if you want
--WHERE NOT EXISTS ( --not exists WHAT?
-- SELECT 1 FROM Table1
--)
order by id OFFSET (#BatchSize * #Counter) ROWS FETCH NEXT #Batchsize ROWS ONLY;
--SET #Counter=#Counter+1
select #rows = ##rowcount, #counter = #counter + 1
END
Related
I have the below query which returns 400 million rows. I want to run the query so it loops through and inserts 1 million records at a time. Please can I get the loop query.
insert into AST (DataAreaId, Name)
select f.DataAreaId, its.Name....... etc
from Transform.InventFin f
inner join Staging.INVENTSETTLEMENT its
on f.ITSRECID=its.RECID
and f.DataAreaId=its.DATAAREAID
Try like following(Assuming that DataAreaId is Unique, if not you need to include those columns in NOT EXISTS).
declare #Count int
set #Count = 1
while #Count > 0
begin
insert into AST (DataAreaId, Name)
select TOP (1000000) f.DataAreaId, its.Name....... etc
from Transform.InventFin f
inner join Staging.INVENTSETTLEMENT its
on f.ITSRECID=its.RECID
and f.DataAreaId=its.DATAAREAID
WHERE NOT EXISTS
(
SELECT 1 FROM AST A WHERE AST.DataAreaId = F.DataAreaId
)
set #Count = ##ROWCOUNT
end
I am trying to loop through my records with a where clause.
I am trying to get only top 100 rows first and then next 100 (some logic i apply with what i get in the select clause)
But, if the first 100 rows does not return a result, it does not goes into 2nd 100 set of data.
My query is :
DECLARE #BatchSize INT = 100
DECLARE #Counter INT = 0
DECLARE #TableCount INT = 0
set #TableCount = (select count(*) from Table1) //#TableCount = 10000
while #Counter < #TableCount/#BatchSize //#Counter < 100
BEGIN
SET #Counter=#Counter+1
INSERT INTO Table4
SELECT TOP(#BatchSize) * FROM Table2
WHERE NOT EXISTS (SELECT * Table3) and some condition
Here, If i dont get the data for first 100 rows, it wont go to next 100 set of data.
What should I do ?
Instead of TOP #BatchSize in the SELECT clause, try OFFSET #BatchSize * #Counter FETCH NEXT #Batchsize ROWS ONLY after the WHERE clause.
Based on comments, you may also want to look into a SELECT INTO query, as well as the nolock query hint.
what is the problem with the #temp variable?
create function dbo.getNumOfReviews2 (#email varchar(40))
returns int
as begin
declare #numOfReviews int
select #numOfReviews = count(*)
from dbo.Reviews
where email = #email
group by Email
return #numOfReviews
end
CREATE TRIGGER setDiscount
ON dbo.[Contains]
FOR INSERT
AS
DECLARE #OrderID int
DECLARE #ProductID int
DECLARE #Size VarChar(15)
DECLARE #temp int
IF CURSOR_STATUS('global','C_CURSOR')>=-1
BEGIN
DEALLOCATE C_CURSOR
END
DECLARE C_CURSOR CURSOR
FOR SELECT ProductID,OrderID,Size
FROM INSERTED
BEGIN
OPEN C_CURSOR
FETCH NEXT FROM C_CURSOR INTO #ProductID,#OrderID,#Size
WHILE (##FETCH_STATUS=0)
BEGIN
#temp = dbo.getNumOfReviews2(select BillingEmail from dbo.Orders where OrderID=#OrderID)
IF (SELECT COUNT(*)
FROM dbo.[Contains]
WHERE OrderID = #OrderID) > 5 or (SELECT sum(Quantity) FROM dbo.[Contains] WHERE OrderID=#OrderID) > 10 or
( #temp )> 5
UPDATE [Contains]
SET [Savings%] = [Savings%] + 0.05
WHERE OrderID = #OrderID and ProductID = #ProductID and Size = #Size
FETCH NEXT FROM C_CURSOR INTO #ProductID,#OrderID,#Size
END
END
Use select to call scalar function
correct way to do this would be
select #temp = dbo.getNumOfReviews2(BillingEmail)
from dbo.Orders
where OrderID=#OrderID
Note: It is not advisable to write big logic inside a trigger. Triggers should be simple and fast otherwise your DML operations will be slow. Moreover you have used a CURSOR which should be avoided at any cost. Rewrite the code using SET based approach.
Here is a SET based approach code
;WITH cte
AS (SELECT c1.orderid
FROM dbo.[contains] c1
INNER JOIN inserted i1
ON i1.orderid = c1.orderid
GROUP BY orderid
HAVING Count(*) > 5
OR Sum(quantity) > 5
OR #temp > 5)
UPDATE C
SET [savings%] = [savings%] + 0.05
FROM [contains] C
INNER JOIN inserted I
ON I.orderid = C.orderid
AND I.productid = C.productid
AND I.size = C.size
AND EXISTS (SELECT 1
FROM cte c1
WHERE c1.orderid = c.orderid)
CREATE OR REPLACE COVID19VMS_VACCINESHOT_T1
BEFORE INSERT OR UPDATE ON VACCINENEXTSHOTDATE
FOR EACH ROW
BEGIN
IF :NEW.VACCINESHOTDATE := 1
:NEW.VACCINEXTSHOTDATE := to_date(VACCINESHOTDATE +28)
END IF;
IF :NEW.VACCINESHOTDATE :=2
:NEW.VACCINENEXTSHOTDATE IS NULL
END IF
END
Can anyone help me to generate a repetitive sequential number using SQL Server 2008. Say I have a table of 1000 rows and a new field (int) added to the table. All I need is to auto fill that particular field with sequential numbers 1-100 all the way to the last row.
I have this but doesnt seem that it is working. You help is much appreciated.
DECLARE #id INT
SET #id = 0
while #id < 101
BEGIN
UPDATE Names SET id=#id
set #id=#id+1
END
USE tempdb
GO
DROP TABLE tableof1000rows
GO
CREATE TABLE tableof1000rows (id int identity(1,1), nb int, value varchar(128))
GO
INSERT INTO tableof1000rows (value)
SELECT TOP 1000 o1.name
FROM sys.objects o1
CROSS JOIN sys.objects o2
GO
UPDATE t1
SET nb = t2.nb
FROM tableof1000rows t1
JOIN (SELECT id, (ROW_NUMBER() OVER (ORDER BY id) % 100) + 1 as nb FROM tableof1000rows) t2 ON t1.id = t2.id
GO
SELECT *
FROM tableof1000rows
Use ROW_NUMBER to generate a number. Use modulo maths to get values from 1 to 100.
go
create table dbo.Demo1
(
DID int not null identity primary key,
RepetitiveSequentialNumber int not null
) ;
go
insert into dbo.Demo1 values ( 0 )
go 1000 -- This is just to get 1,000 rows into the table.
-- Get a sequential number.
select DID, row_number() over ( order by DID ) as RSN
into #RowNumbers
from dbo.Demo1 ;
-- Take the last two digits. This gives us values from 0 to 99.
update #RowNumbers
set RSN = RSN % 100 ;
-- Change the 0 values to 100.
update #RowNumbers
set RSN = case when RSN = 0 then 100 else RSN end ;
-- Update the main table.
update dbo.Demo1
set RepetitiveSequentialNumber = r.RSN
from dbo.Demo1 as d inner join #RowNumbers as r on r.DID = d.DID ;
select *
from dbo.Demo1 ;
Not pretty or elegant, but..
while exists(select * from tbl where new_col is null)
update top(1) tbl
set new_col=(select ISNULL(max(new_col),0)+1 from tbl WHERE new_col is null)
Your way isn't working because you are trying to set a value rather than insert one. I'm sure you have found a solution by now but if not then try this instead.
DECLARE #id INT
SET #id = 0
while #id < 101
BEGIN
INSERT Names select #id
set #id=#id+1
END
I have two tables with overlaping data. One table is about 90% duplicate of the other. I need to identify the 10% of unique records in a table and move it to it's parent table. Both of these tables are 400 million + rows with 300+ columns. The method I am attempting is adding a flag field to uniquely ID the records I need to transfer however I need to update the field and am struggling with the logic. Below is what I have put together so far and it causes a never ending loop. There are no null values in either table.
Declare #counter int
Declare #RowsEffected int
Declare #RowsCnt int
Declare #Err int
SELECT #COUNTER = 1
SELECT #RowsEffected = 0
while (#counter > 0)
begin
set Rowcount 10000000
update Table1
set Existsflg = 1
where exists (
Select Fields
from Table1
Except
Select Fields
from table2 )
Select #RowsCnt = ##ROWCOUNT , #Err = ##ERROR
If #Err <> 0
begin
Print 'Problem Updating the records'
end
IF #RowsCnt = 0
SELECT #COUNTER = 0
ELSE
SELECT #RowsEffected = #RowsEffected + #RowsCnt
PRINT 'The total number of rows effected :'+convert(varchar,#RowsEffected)
WAITFOR DELAY '00:00:10'
END
SET ROWCOUNT 0
Go
Thanks!
This is how I did it one time.
I didn't use RowCount, I used Select TOP (N) and "while exists"
My "source" dbo.Employee table was on another server.
GO
USE [$(DestinationDatabaseName)]
GO
/*
READ ME !!!
Replace
$(SourceServer).$(SourceDatabaseName)
With (the Server and DatabaseName of the SOURCE data)
(ex:) [OtherServer].[OtherDatabase]
*/
--SubFolder: SQLReplicateReplacer
print '[uspEmployeeReplicateReplacer]'
go
IF EXISTS (SELECT * FROM sys.objects WHERE object_id = OBJECT_ID(N'[dbo].[uspEmployeeReplicateReplacer]') AND type in (N'P', N'PC'))
DROP PROCEDURE [dbo].[uspEmployeeReplicateReplacer]
Go
/*
declare #numberRowsAffected int
declare #ErrorNumber int
exec [dbo].[uspEmployeeReplicateReplacer] #numberRowsAffected output , #ErrorNumber output
print #numberRowsAffected
print #ErrorNumber
print ''
*/
CREATE PROCEDURE [dbo].[uspEmployeeReplicateReplacer] (
#numberRowsAffected int output --return
,
#ErrorNumber int output
)
AS
SET NOCOUNT ON
select #ErrorNumber = 0
declare #ErrorTracker int
declare #insertRowCount int
declare #updateRowCount int
select #insertRowCount = 0
select #updateRowCount = 0
IF OBJECT_ID('tempdb..#Employeeupdate') IS NOT NULL
begin
drop table #Employeeupdate
end
CREATE TABLE #Employeeupdate (
EmployeeKeyID int IDENTITY (1,1),
EmployeeUUID uniqueidentifier,
EmployeeLabel varchar(64),
EmployeeDescription varchar(128)
)
declare #ManualReplicationRowCount int
/* I put this value in a stored procedure, so I could change it in one place */
/* EXEC dbo.uspInternalSettingGetManualReplicationRowCount #ManualReplicationRowCount output */
Select #ManualReplicationRowCount = 1000
declare #MaximumLoopCounter int
select #MaximumLoopCounter = 10000
while (#MaximumLoopCounter > 0) and exists
(
Select
TOP 1 null
from [$(SourceServer)].[$(SourceDatabaseName)].dbo.Employee vart with (nolock)
where not exists
(
select null from dbo.Employee with (nolock) -- destinationTable
Where
/*
destinationTable.SOMEUNIQUECOLUMN1 = sourceTable.SOMEUNIQUECOLUMN1
and
destinationTable.SOMEUNIQUECOLUMN2 = sourceTable.SOMEUNIQUECOLUMN2
*/
dbo.Employee.EmployeeUUID = vart.EmployeeUUID
)
)
BEGIN
select #MaximumLoopCounter = #MaximumLoopCounter - 1
DELETE FROM #Employeeupdate
Insert into #Employeeupdate
(
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
)
Select
TOP (#ManualReplicationRowCount)
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
from [$(SourceServer)].[$(SourceDatabaseName)].dbo.Employee vart with (nolock)
where not exists
(
select null from dbo.Employee with (nolock) -- destinationTable
Where
/*
destinationTable.SOMEUNIQUECOLUMN1 = sourceTable.SOMEUNIQUECOLUMN1
and
destinationTable.SOMEUNIQUECOLUMN2 = sourceTable.SOMEUNIQUECOLUMN2
*/
dbo.Employee.EmployeeUUID = vart.EmployeeUUID
)
SET NOCOUNT OFF
Insert into dbo.Employee
(
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
)
Select
EmployeeUUID,
EmployeeLabel,
EmployeeDescription
from
#Employeeupdate
SELECT #insertRowCount = ##ROWCOUNT , #ErrorTracker = ##ERROR
if #ErrorTracker <> 0
BEGIN
select #ErrorNumber = #ErrorTracker
select #MaximumLoopCounter = 0 --Bail Out !!!
END
SET NOCOUNT ON
END --End While Loop
/*
SET NOCOUNT OFF
Update dbo.Employee
Set
--EmployeeUUID = vart.EmployeeUUID,
EmployeeLabel = vart.EmployeeLabel,
EmployeeDescription = vart.EmployeeDescription
From
dbo.Employee with (nolock) , [$(SourceServer)].[$(SourceDatabaseName)].dbo.Employee vart with (nolock)
Where
--Relationship
dbo.Employee.EmployeeUUID = vart.EmployeeUUID
SELECT #updateRowCount = ##ROWCOUNT
SET NOCOUNT ON
*/
SELECT #numberRowsAffected = #insertRowCount + #updateRowCount
print '/#Employeeupdate COUNT/'
print #numberRowsAffected
print '-------------------------'
IF OBJECT_ID('tempdb..#Employeeupdate') IS NOT NULL
begin
drop table #Employeeupdate
end
SET NOCOUNT OFF
GO
GRANT EXECUTE ON dbo.uspEmployeeReplicateReplacer TO $(DBUSERNAME)
GO
I’d suggest you do this in batches of 1M-5M at a time because you have a ton of data to update.
What I would do in this case is:
a) add new column named Processed (bit) that would be updated for all rows that are processed
b) select 1M rows into temp table (this may not be needed but it will make things a bit
cleaner)
c) insert all non-duplicate records into other table
d) update rows and mark them as processes